You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2018/09/07 13:49:04 UTC
[01/50] [abbrv] ignite git commit: IGNITE-9424 Set proper partition
for key during insert - Fixes #4645.
Repository: ignite
Updated Branches:
refs/heads/ignite-5960 8531961f9 -> a7ee714e6
IGNITE-9424 Set proper partition for key during insert - Fixes #4645.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/dcefc16b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/dcefc16b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/dcefc16b
Branch: refs/heads/ignite-5960
Commit: dcefc16b6990eccea407dc29c81429b615bbd7fe
Parents: 145a289
Author: Anton Kalashnikov <ka...@yandex.ru>
Authored: Fri Aug 31 12:21:43 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Aug 31 12:21:43 2018 +0300
----------------------------------------------------------------------
.../cache/IgniteCacheOffheapManagerImpl.java | 24 ++++++++++++++++----
1 file changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/dcefc16b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 2fe097c..91c6f3e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -101,6 +101,7 @@ import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteClosure;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgnitePredicate;
+import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
@@ -1646,7 +1647,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
@Nullable CacheDataRow oldRow) throws IgniteCheckedException {
int cacheId = grp.storeCacheIdInDataPage() ? cctx.cacheId() : CU.UNDEFINED_CACHE_ID;
- DataRow dataRow = new DataRow(key, val, ver, partId, expireTime, cacheId);
+ DataRow dataRow = makeDataRow(key, val, ver, expireTime, cacheId);
if (canUpdateOldRow(cctx, oldRow, dataRow) && rowStore.updateRow(oldRow.link(), dataRow))
dataRow.link(oldRow.link());
@@ -1667,6 +1668,22 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
return dataRow;
}
+ /**
+ * @param key Cache key.
+ * @param val Cache value.
+ * @param ver Version.
+ * @param expireTime Expired time.
+ * @param cacheId Cache id.
+ * @return Made data row.
+ */
+ @NotNull private DataRow makeDataRow(KeyCacheObject key, CacheObject val, GridCacheVersion ver, long expireTime,
+ int cacheId) {
+ if (key.partition() == -1)
+ key.partition(partId);
+
+ return new DataRow(key, val, ver, partId, expireTime, cacheId);
+ }
+
/** {@inheritDoc} */
@Override public boolean mvccInitialValue(
GridCacheContext cctx,
@@ -2342,10 +2359,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
assert oldRow == null || oldRow.cacheId() == cacheId : oldRow;
- if (key.partition() == -1)
- key.partition(partId);
-
- DataRow dataRow = new DataRow(key, val, ver, partId, expireTime, cacheId);
+ DataRow dataRow = makeDataRow(key, val, ver, expireTime, cacheId);
CacheObjectContext coCtx = cctx.cacheObjectContext();
[14/50] [abbrv] ignite git commit: IGNITE-9425 Fixed NPE on index
rebuild - Fixes #4647.
Posted by ag...@apache.org.
IGNITE-9425 Fixed NPE on index rebuild - Fixes #4647.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c7aaee7e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c7aaee7e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c7aaee7e
Branch: refs/heads/ignite-5960
Commit: c7aaee7ed577dbfc550948494ca74d8261bbdff4
Parents: 8cb4591
Author: dkarachentsev <dk...@gridgain.com>
Authored: Mon Sep 3 19:45:44 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Sep 3 19:45:44 2018 +0300
----------------------------------------------------------------------
.../GridCacheDatabaseSharedManager.java | 3 +-
.../IgniteTwoRegionsRebuildIndexTest.java | 124 +++++++++++++++++++
.../IgnitePdsWithIndexingTestSuite.java | 2 +
3 files changed, 128 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7aaee7e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 2e5f26f..a86b9f5 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1362,7 +1362,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
final int cacheId = cacheCtx.cacheId();
final GridFutureAdapter<Void> usrFut = idxRebuildFuts.get(cacheId);
- if (!cctx.pageStore().hasIndexStore(cacheCtx.groupId()) && cacheCtx.affinityNode()) {
+ if (!cctx.pageStore().hasIndexStore(cacheCtx.groupId()) && cacheCtx.affinityNode()
+ && cacheCtx.group().persistenceEnabled()) {
IgniteInternalFuture<?> rebuildFut = cctx.kernalContext().query()
.rebuildIndexesFromHash(Collections.singleton(cacheCtx.cacheId()));
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7aaee7e/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteTwoRegionsRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteTwoRegionsRebuildIndexTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteTwoRegionsRebuildIndexTest.java
new file mode 100644
index 0000000..7d5b296
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteTwoRegionsRebuildIndexTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests the case when preformed index rebuild for created by client in-memory cache.
+ */
+public class IgniteTwoRegionsRebuildIndexTest extends GridCommonAbstractTest {
+ /** */
+ private static final String PERSISTED_CACHE = "persisted";
+
+ /** */
+ private static final String INMEMORY_CACHE = "inmemory";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ boolean client = igniteInstanceName.startsWith("client");
+
+ DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+
+ if (!client) {
+ DataRegionConfiguration drCfg1 = new DataRegionConfiguration();
+ drCfg1.setMaxSize(16 * 1024 * 1024);
+ drCfg1.setName("nopersistence");
+ drCfg1.setInitialSize(drCfg1.getMaxSize());
+ drCfg1.setPersistenceEnabled(false);
+
+ DataRegionConfiguration drCfg2 = new DataRegionConfiguration();
+ drCfg2.setMaxSize(16 * 1024 * 1024);
+ drCfg2.setName("persistence");
+ drCfg2.setInitialSize(drCfg2.getMaxSize());
+ drCfg2.setPersistenceEnabled(true);
+
+ dsCfg.setDataRegionConfigurations(drCfg1, drCfg2);
+
+ cfg.setDataStorageConfiguration(dsCfg);
+ }
+ else {
+ CacheConfiguration ccfg1 = new CacheConfiguration(PERSISTED_CACHE);
+ CacheConfiguration ccfg2 = new CacheConfiguration(INMEMORY_CACHE);
+
+ ccfg1.setDataRegionName("persistence");
+ ccfg2.setDataRegionName("nopersistence");
+
+ cfg.setCacheConfiguration(ccfg1, ccfg2);
+ cfg.setClientMode(true);
+ }
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRebuildIndexes() throws Exception {
+ startGrid("server");
+ Ignite client = startGrid("client");
+
+ client.cluster().active(true);
+
+ populateData(client, PERSISTED_CACHE);
+ populateData(client, INMEMORY_CACHE);
+
+ stopGrid("server");
+ startGrid("server");
+
+ stopGrid("client");
+ startGrid("client");
+ }
+
+ /**
+ * @param ignite Ignite.
+ * @param cacheName Cache name.
+ */
+ private void populateData(Ignite ignite, String cacheName) {
+ try (IgniteDataStreamer<Object, Object> streamer = ignite.dataStreamer(cacheName)) {
+ for (int i = 0; i < 1000; i++)
+ streamer.addData(i, i);
+
+ streamer.flush();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7aaee7e/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
index 033e5a9..67b9fad 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
@@ -25,6 +25,7 @@ import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithInde
import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingWalRestoreTest;
import org.apache.ignite.internal.processors.database.IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest;
import org.apache.ignite.internal.processors.database.IgnitePersistentStoreSchemaLoadTest;
+import org.apache.ignite.internal.processors.database.IgniteTwoRegionsRebuildIndexTest;
/**
*
@@ -44,6 +45,7 @@ public class IgnitePdsWithIndexingTestSuite extends TestSuite {
suite.addTestSuite(IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.class);
suite.addTestSuite(IgnitePersistentStoreSchemaLoadTest.class);
suite.addTestSuite(IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest.class);
+ suite.addTestSuite(IgniteTwoRegionsRebuildIndexTest.class);
return suite;
}
[19/50] [abbrv] ignite git commit: IGNITE-9361 Remove
IgniteInternalCache.isMongo*Cache() and other remnants. - Fixes #4678.
Posted by ag...@apache.org.
IGNITE-9361 Remove IgniteInternalCache.isMongo*Cache() and other remnants. - Fixes #4678.
Signed-off-by: Dmitriy Govorukhin <dm...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/de30a86e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/de30a86e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/de30a86e
Branch: refs/heads/ignite-5960
Commit: de30a86e740067ed239602fb5dc28396144fd446
Parents: 6e7b86e
Author: Ilya Kasnacheev <il...@gmail.com>
Authored: Tue Sep 4 13:27:28 2018 +0300
Committer: Dmitriy Govorukhin <dm...@gmail.com>
Committed: Tue Sep 4 13:27:28 2018 +0300
----------------------------------------------------------------------
.../org/apache/ignite/internal/GridTopic.java | 3 ---
.../ignite/internal/IgniteNodeAttributes.java | 3 ---
.../processors/cache/GridCacheAdapter.java | 18 ---------------
.../processors/cache/GridCacheProxyImpl.java | 24 --------------------
.../processors/cache/IgniteInternalCache.java | 14 ------------
.../distributed/near/GridNearCacheAdapter.java | 10 --------
modules/core/src/test/config/tests.properties | 5 ----
.../cache/hibernate/HibernateCacheProxy.java | 10 --------
8 files changed, 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/de30a86e/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index 68cea4f..5610b2a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -82,9 +82,6 @@ public enum GridTopic {
TOPIC_CONTINUOUS,
/** */
- TOPIC_MONGO,
-
- /** */
TOPIC_TIME_SYNC,
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/de30a86e/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
index 61a98a1..539727d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
@@ -82,9 +82,6 @@ public final class IgniteNodeAttributes {
public static final String ATTR_IGFS = ATTR_PREFIX + ".igfs";
/** Internal attribute name constant. */
- public static final String ATTR_MONGO = ATTR_PREFIX + ".mongo";
-
- /** Internal attribute name constant. */
public static final String ATTR_DAEMON = ATTR_PREFIX + ".daemon";
/** Internal attribute name constant. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/de30a86e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 159d681..4481e4d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -276,14 +276,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/** Whether this cache is IGFS data cache. */
private boolean igfsDataCache;
- /** Whether this cache is Mongo data cache. */
- @SuppressWarnings("UnusedDeclaration")
- private boolean mongoDataCache;
-
- /** Whether this cache is Mongo meta cache. */
- @SuppressWarnings("UnusedDeclaration")
- private boolean mongoMetaCache;
-
/** Current IGFS data cache size. */
private LongAdder igfsDataCacheSize;
@@ -4572,16 +4564,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
return igfsDataCacheSize.longValue();
}
- /** {@inheritDoc} */
- @Override public boolean isMongoDataCache() {
- return mongoDataCache;
- }
-
- /** {@inheritDoc} */
- @Override public boolean isMongoMetaCache() {
- return mongoMetaCache;
- }
-
/**
* Callback invoked when data is added to IGFS cache.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/de30a86e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java
index fb0a556..a9ce448 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java
@@ -445,30 +445,6 @@ public class GridCacheProxyImpl<K, V> implements IgniteInternalCache<K, V>, Exte
}
/** {@inheritDoc} */
- @Override public boolean isMongoDataCache() {
- CacheOperationContext prev = gate.enter(opCtx);
-
- try {
- return delegate.isMongoDataCache();
- }
- finally {
- gate.leave(prev);
- }
- }
-
- /** {@inheritDoc} */
- @Override public boolean isMongoMetaCache() {
- CacheOperationContext prev = gate.enter(opCtx);
-
- try {
- return delegate.isMongoMetaCache();
- }
- finally {
- gate.leave(prev);
- }
- }
-
- /** {@inheritDoc} */
@Override public Map<K, V> getAll(@Nullable Collection<? extends K> keys) throws IgniteCheckedException {
CacheOperationContext prev = gate.enter(opCtx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/de30a86e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java
index bbedef8..cba2228 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java
@@ -1658,20 +1658,6 @@ public interface IgniteInternalCache<K, V> extends Iterable<Cache.Entry<K, V>> {
public long igfsDataSpaceUsed();
/**
- * Checks whether this cache is Mongo data cache.
- *
- * @return {@code True} if this cache is mongo data cache.
- */
- public boolean isMongoDataCache();
-
- /**
- * Checks whether this cache is Mongo meta cache.
- *
- * @return {@code True} if this cache is mongo meta cache.
- */
- public boolean isMongoMetaCache();
-
- /**
* @param keepBinary Keep binary flag.
* @param p Optional key/value predicate.
* @return Scan query iterator.
http://git-wip-us.apache.org/repos/asf/ignite/blob/de30a86e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
index 672693c..3904781 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java
@@ -396,16 +396,6 @@ public abstract class GridNearCacheAdapter<K, V> extends GridDistributedCacheAda
}
/** {@inheritDoc} */
- @Override public boolean isMongoDataCache() {
- return dht().isMongoDataCache();
- }
-
- /** {@inheritDoc} */
- @Override public boolean isMongoMetaCache() {
- return dht().isMongoMetaCache();
- }
-
- /** {@inheritDoc} */
@Override public List<GridCacheClearAllRunnable<K, V>> splitClearLocally(boolean srv, boolean near,
boolean readers) {
assert configuration().getNearConfiguration() != null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/de30a86e/modules/core/src/test/config/tests.properties
----------------------------------------------------------------------
diff --git a/modules/core/src/test/config/tests.properties b/modules/core/src/test/config/tests.properties
index 718d661..b39e3ca 100644
--- a/modules/core/src/test/config/tests.properties
+++ b/modules/core/src/test/config/tests.properties
@@ -146,8 +146,3 @@ ssl.keystore.trustboth.path=@{IGNITE_HOME}/modules/clients/src/test/keystore/ca/
# Hadoop home directory.
hadoop.home=@{HADOOP_HOME}
-
-# Sharded mongo properties
-mongos.host=192.168.2.10
-mongos.port=27017
-sharded.db.name=gg-test-db
http://git-wip-us.apache.org/repos/asf/ignite/blob/de30a86e/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
----------------------------------------------------------------------
diff --git a/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java b/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
index 70e55de..fdb87f0 100644
--- a/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
+++ b/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
@@ -602,16 +602,6 @@ public class HibernateCacheProxy implements IgniteInternalCache<Object, Object>
}
/** {@inheritDoc} */
- @Override public boolean isMongoDataCache() {
- return delegate.isMongoDataCache();
- }
-
- /** {@inheritDoc} */
- @Override public boolean isMongoMetaCache() {
- return delegate.isMongoMetaCache();
- }
-
- /** {@inheritDoc} */
@Nullable @Override public ExpiryPolicy expiry() {
return delegate.expiry();
}
[12/50] [abbrv] ignite git commit: IGNITE-584 Fixed scan query to
return consistent results on changing topology - Fixes #4351.
Posted by ag...@apache.org.
IGNITE-584 Fixed scan query to return consistent results on changing topology - Fixes #4351.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4153b74e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4153b74e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4153b74e
Branch: refs/heads/ignite-5960
Commit: 4153b74e3c77ef8bbdd5ed99bab4db615390cab5
Parents: 4c27320
Author: Evgeny Stanilovskiy <es...@gridgain.com>
Authored: Mon Sep 3 16:21:58 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Sep 3 16:21:58 2018 +0300
----------------------------------------------------------------------
.../ignite/internal/processors/cache/GridCacheMapEntry.java | 2 +-
.../processors/datastructures/GridSetQueryPredicate.java | 4 ++--
.../cache/datastructures/GridCacheSetAbstractSelfTest.java | 4 ----
.../datastructures/GridCacheSetFailoverAbstractSelfTest.java | 6 ++----
.../GridCachePartitionedAtomicSetFailoverSelfTest.java | 2 +-
.../partitioned/IgnitePartitionedSetNoBackupsSelfTest.java | 2 +-
6 files changed, 7 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4153b74e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 4dc4944..8c2b939 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -3357,7 +3357,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
null,
topVer);
- cctx.dataStructures().onEntryUpdated(key, false, true);
+ cctx.dataStructures().onEntryUpdated(key, false, false);
}
onUpdateFinished(updateCntr);
http://git-wip-us.apache.org/repos/asf/ignite/blob/4153b74e/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
index bc6c182..2de3dec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridSetQueryPredicate.java
@@ -99,7 +99,7 @@ public class GridSetQueryPredicate<K, V> implements IgniteBiPredicate<K, V>, Ext
*/
private boolean filterKeys() {
return !collocated && !(ctx.isLocal() || ctx.isReplicated()) &&
- (ctx.config().getBackups() > 0 || CU.isNearEnabled(ctx));
+ (CU.isNearEnabled(ctx) || ctx.isPartitioned());
}
/** {@inheritDoc} */
@@ -118,4 +118,4 @@ public class GridSetQueryPredicate<K, V> implements IgniteBiPredicate<K, V>, Ext
@Override public String toString() {
return S.toString(GridSetQueryPredicate.class, this);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4153b74e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetAbstractSelfTest.java
index 9a707eb..0ed4a97 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetAbstractSelfTest.java
@@ -582,8 +582,6 @@ public abstract class GridCacheSetAbstractSelfTest extends IgniteCollectionAbstr
if (collectionCacheMode() == LOCAL)
return;
- fail("https://issues.apache.org/jira/browse/IGNITE-584");
-
testNodeJoinsAndLeaves(false);
}
@@ -594,8 +592,6 @@ public abstract class GridCacheSetAbstractSelfTest extends IgniteCollectionAbstr
if (collectionCacheMode() == LOCAL)
return;
- fail("https://issues.apache.org/jira/browse/IGNITE-584");
-
testNodeJoinsAndLeaves(true);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4153b74e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetFailoverAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetFailoverAbstractSelfTest.java
index f8af2a2..ce320bd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheSetFailoverAbstractSelfTest.java
@@ -116,8 +116,7 @@ public abstract class GridCacheSetFailoverAbstractSelfTest extends IgniteCollect
try {
int size = set.size();
- // TODO: IGNITE-584, check for equality when IGNITE-584 fixed.
- assertTrue(size > 0);
+ assertEquals(ITEMS, size);
}
catch (IgniteException ignore) {
// No-op.
@@ -134,8 +133,7 @@ public abstract class GridCacheSetFailoverAbstractSelfTest extends IgniteCollect
cnt++;
}
- // TODO: IGNITE-584, check for equality when IGNITE-584 fixed.
- assertTrue(cnt > 0);
+ assertEquals(ITEMS, cnt);
}
catch (IgniteException ignore) {
// No-op.
http://git-wip-us.apache.org/repos/asf/ignite/blob/4153b74e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSetFailoverSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSetFailoverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSetFailoverSelfTest.java
index c2af2b1..4673549 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSetFailoverSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedAtomicSetFailoverSelfTest.java
@@ -35,4 +35,4 @@ public class GridCachePartitionedAtomicSetFailoverSelfTest extends GridCacheSetF
@Override public void testNodeRestart() throws Exception {
fail("https://issues.apache.org/jira/browse/IGNITE-170");
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4153b74e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/IgnitePartitionedSetNoBackupsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/IgnitePartitionedSetNoBackupsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/IgnitePartitionedSetNoBackupsSelfTest.java
index 4daaeca..5f09dfa 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/IgnitePartitionedSetNoBackupsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/IgnitePartitionedSetNoBackupsSelfTest.java
@@ -70,4 +70,4 @@ public class IgnitePartitionedSetNoBackupsSelfTest extends GridCachePartitionedS
}
}
}
-}
\ No newline at end of file
+}
[35/50] [abbrv] ignite git commit: IGNITE-9412: [ML] GDB convergence
by error support.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/dataset/primitive/FeatureMatrixWithLabelsOnHeapDataBuilder.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/primitive/FeatureMatrixWithLabelsOnHeapDataBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/primitive/FeatureMatrixWithLabelsOnHeapDataBuilder.java
new file mode 100644
index 0000000..be1724c
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/primitive/FeatureMatrixWithLabelsOnHeapDataBuilder.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.dataset.primitive;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import org.apache.ignite.ml.dataset.PartitionDataBuilder;
+import org.apache.ignite.ml.dataset.UpstreamEntry;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+
+/**
+ * A partition {@code data} builder that makes {@link DecisionTreeData}.
+ *
+ * @param <K> Type of a key in <tt>upstream</tt> data.
+ * @param <V> Type of a value in <tt>upstream</tt> data.
+ * @param <C> Type of a partition <tt>context</tt>.
+ */
+public class FeatureMatrixWithLabelsOnHeapDataBuilder<K, V, C extends Serializable>
+ implements PartitionDataBuilder<K, V, C, FeatureMatrixWithLabelsOnHeapData> {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 6273736987424171813L;
+
+ /** Function that extracts features from an {@code upstream} data. */
+ private final IgniteBiFunction<K, V, Vector> featureExtractor;
+
+ /** Function that extracts labels from an {@code upstream} data. */
+ private final IgniteBiFunction<K, V, Double> lbExtractor;
+
+ /**
+ * Constructs a new instance of decision tree data builder.
+ *
+ * @param featureExtractor Function that extracts features from an {@code upstream} data.
+ * @param lbExtractor Function that extracts labels from an {@code upstream} data.
+ */
+ public FeatureMatrixWithLabelsOnHeapDataBuilder(IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+ this.featureExtractor = featureExtractor;
+ this.lbExtractor = lbExtractor;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FeatureMatrixWithLabelsOnHeapData build(Iterator<UpstreamEntry<K, V>> upstreamData, long upstreamDataSize, C ctx) {
+ double[][] features = new double[Math.toIntExact(upstreamDataSize)][];
+ double[] labels = new double[Math.toIntExact(upstreamDataSize)];
+
+ int ptr = 0;
+ while (upstreamData.hasNext()) {
+ UpstreamEntry<K, V> entry = upstreamData.next();
+
+ features[ptr] = featureExtractor.apply(entry.getKey(), entry.getValue()).asArray();
+
+ labels[ptr] = lbExtractor.apply(entry.getKey(), entry.getValue());
+
+ ptr++;
+ }
+
+ return new FeatureMatrixWithLabelsOnHeapData(features, labels);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBOnTreesLearningStrategy.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBOnTreesLearningStrategy.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBOnTreesLearningStrategy.java
index 8589a79..6ebbda1 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBOnTreesLearningStrategy.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBOnTreesLearningStrategy.java
@@ -17,12 +17,13 @@
package org.apache.ignite.ml.tree.boosting;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.ignite.ml.Model;
import org.apache.ignite.ml.composition.ModelsComposition;
import org.apache.ignite.ml.composition.boosting.GDBLearningStrategy;
+import org.apache.ignite.ml.composition.boosting.GDBTrainer;
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceChecker;
import org.apache.ignite.ml.composition.predictionsaggregator.WeightedPredictionsAggregator;
import org.apache.ignite.ml.dataset.Dataset;
import org.apache.ignite.ml.dataset.DatasetBuilder;
@@ -54,22 +55,30 @@ public class GDBOnTreesLearningStrategy extends GDBLearningStrategy {
}
/** {@inheritDoc} */
- @Override public <K, V> List<Model<Vector, Double>> learnModels(DatasetBuilder<K, V> datasetBuilder,
- IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+ @Override public <K, V> List<Model<Vector, Double>> update(GDBTrainer.GDBModel mdlToUpdate,
+ DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
DatasetTrainer<? extends Model<Vector, Double>, Double> trainer = baseMdlTrainerBuilder.get();
assert trainer instanceof DecisionTree;
DecisionTree decisionTreeTrainer = (DecisionTree) trainer;
- List<Model<Vector, Double>> models = new ArrayList<>();
+ List<Model<Vector, Double>> models = initLearningState(mdlToUpdate);
+
+ ConvergenceChecker<K,V> convCheck = checkConvergenceStgyFactory.create(sampleSize,
+ externalLbToInternalMapping, loss, datasetBuilder, featureExtractor, lbExtractor);
+
try (Dataset<EmptyContext, DecisionTreeData> dataset = datasetBuilder.build(
new EmptyContextBuilder<>(),
new DecisionTreeDataBuilder<>(featureExtractor, lbExtractor, useIndex)
)) {
for (int i = 0; i < cntOfIterations; i++) {
- double[] weights = Arrays.copyOf(compositionWeights, i);
+ double[] weights = Arrays.copyOf(compositionWeights, models.size());
WeightedPredictionsAggregator aggregator = new WeightedPredictionsAggregator(weights, meanLabelValue);
- Model<Vector, Double> currComposition = new ModelsComposition(models, aggregator);
+ ModelsComposition currComposition = new ModelsComposition(models, aggregator);
+
+ if(convCheck.isConverged(dataset, currComposition))
+ break;
dataset.compute(part -> {
if(part.getCopyOfOriginalLabels() == null)
@@ -78,7 +87,7 @@ public class GDBOnTreesLearningStrategy extends GDBLearningStrategy {
for(int j = 0; j < part.getLabels().length; j++) {
double mdlAnswer = currComposition.apply(VectorUtils.of(part.getFeatures()[j]));
double originalLbVal = externalLbToInternalMapping.apply(part.getCopyOfOriginalLabels()[j]);
- part.getLabels()[j] = -lossGradient.apply(sampleSize, originalLbVal, mdlAnswer);
+ part.getLabels()[j] = -loss.gradient(sampleSize, originalLbVal, mdlAnswer);
}
});
@@ -92,6 +101,7 @@ public class GDBOnTreesLearningStrategy extends GDBLearningStrategy {
throw new RuntimeException(e);
}
+ compositionWeights = Arrays.copyOf(compositionWeights, models.size());
return models;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
index d5750ea..b8a16dc 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
@@ -19,18 +19,14 @@ package org.apache.ignite.ml.tree.data;
import java.util.ArrayList;
import java.util.List;
+import org.apache.ignite.ml.dataset.primitive.FeatureMatrixWithLabelsOnHeapData;
import org.apache.ignite.ml.tree.TreeFilter;
/**
- * A partition {@code data} of the containing matrix of features and vector of labels stored in heap.
+ * A partition {@code data} of the containing matrix of features and vector of labels stored in heap
+ * with index on features.
*/
-public class DecisionTreeData implements AutoCloseable {
- /** Matrix with features. */
- private final double[][] features;
-
- /** Vector with labels. */
- private final double[] labels;
-
+public class DecisionTreeData extends FeatureMatrixWithLabelsOnHeapData implements AutoCloseable {
/** Copy of vector with original labels. Auxiliary for Gradient Boosting on Trees.*/
private double[] copyOfOriginalLabels;
@@ -48,10 +44,7 @@ public class DecisionTreeData implements AutoCloseable {
* @param buildIdx Build index.
*/
public DecisionTreeData(double[][] features, double[] labels, boolean buildIdx) {
- assert features.length == labels.length : "Features and labels have to be the same length";
-
- this.features = features;
- this.labels = labels;
+ super(features, labels);
this.buildIndex = buildIdx;
indexesCache = new ArrayList<>();
@@ -68,6 +61,8 @@ public class DecisionTreeData implements AutoCloseable {
public DecisionTreeData filter(TreeFilter filter) {
int size = 0;
+ double[][] features = getFeatures();
+ double[] labels = getLabels();
for (int i = 0; i < features.length; i++)
if (filter.test(features[i]))
size++;
@@ -95,12 +90,15 @@ public class DecisionTreeData implements AutoCloseable {
* @param col Column.
*/
public void sort(int col) {
- sort(col, 0, features.length - 1);
+ sort(col, 0, getFeatures().length - 1);
}
/** */
private void sort(int col, int from, int to) {
if (from < to) {
+ double[][] features = getFeatures();
+ double[] labels = getLabels();
+
double pivot = features[(from + to) / 2][col];
int i = from, j = to;
@@ -131,19 +129,11 @@ public class DecisionTreeData implements AutoCloseable {
}
/** */
- public double[][] getFeatures() {
- return features;
- }
-
- /** */
- public double[] getLabels() {
- return labels;
- }
-
public double[] getCopyOfOriginalLabels() {
return copyOfOriginalLabels;
}
+ /** */
public void setCopyOfOriginalLabels(double[] copyOfOriginalLabels) {
this.copyOfOriginalLabels = copyOfOriginalLabels;
}
@@ -170,7 +160,7 @@ public class DecisionTreeData implements AutoCloseable {
if (depth == indexesCache.size()) {
if (depth == 0)
- indexesCache.add(new TreeDataIndex(features, labels));
+ indexesCache.add(new TreeDataIndex(getFeatures(), getLabels()));
else {
TreeDataIndex lastIndex = indexesCache.get(depth - 1);
indexesCache.add(lastIndex.filter(filter));
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java
index 3e340f6..89b8c9c 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java
@@ -22,11 +22,13 @@ import java.util.Map;
import java.util.function.BiFunction;
import org.apache.ignite.ml.Model;
import org.apache.ignite.ml.composition.ModelsComposition;
+import org.apache.ignite.ml.composition.boosting.convergence.mean.MeanAbsValueConvergenceCheckerFactory;
+import org.apache.ignite.ml.composition.boosting.convergence.simple.ConvergenceCheckerStubFactory;
import org.apache.ignite.ml.composition.predictionsaggregator.WeightedPredictionsAggregator;
import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
-import org.apache.ignite.ml.trainers.DatasetTrainer;
import org.apache.ignite.ml.tree.DecisionTreeConditionalNode;
import org.apache.ignite.ml.tree.boosting.GDBBinaryClassifierOnTreesTrainer;
import org.apache.ignite.ml.tree.boosting.GDBRegressionOnTreesTrainer;
@@ -54,8 +56,8 @@ public class GDBTrainerTest {
learningSample.put(i, new double[] {xs[i], ys[i]});
}
- DatasetTrainer<ModelsComposition, Double> trainer
- = new GDBRegressionOnTreesTrainer(1.0, 2000, 3, 0.0).withUseIndex(true);
+ GDBTrainer trainer = new GDBRegressionOnTreesTrainer(1.0, 2000, 3, 0.0)
+ .withUseIndex(true);
Model<Vector, Double> mdl = trainer.fit(
learningSample, 1,
@@ -74,7 +76,6 @@ public class GDBTrainerTest {
assertEquals(0.0, mse, 0.0001);
- assertTrue(mdl instanceof ModelsComposition);
ModelsComposition composition = (ModelsComposition)mdl;
assertTrue(composition.toString().length() > 0);
assertTrue(composition.toString(true).length() > 0);
@@ -84,6 +85,13 @@ public class GDBTrainerTest {
assertEquals(2000, composition.getModels().size());
assertTrue(composition.getPredictionsAggregator() instanceof WeightedPredictionsAggregator);
+
+ trainer = trainer.withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.1));
+ assertTrue(trainer.fit(
+ learningSample, 1,
+ (k, v) -> VectorUtils.of(v[0]),
+ (k, v) -> v[1]
+ ).getModels().size() < 2000);
}
/** */
@@ -107,7 +115,7 @@ public class GDBTrainerTest {
}
/** */
- private void testClassifier(BiFunction<GDBBinaryClassifierOnTreesTrainer, Map<Integer, double[]>,
+ private void testClassifier(BiFunction<GDBTrainer, Map<Integer, double[]>,
Model<Vector, Double>> fitter) {
int sampleSize = 100;
double[] xs = new double[sampleSize];
@@ -122,8 +130,9 @@ public class GDBTrainerTest {
for (int i = 0; i < sampleSize; i++)
learningSample.put(i, new double[] {xs[i], ys[i]});
- GDBBinaryClassifierOnTreesTrainer trainer
- = new GDBBinaryClassifierOnTreesTrainer(0.3, 500, 3, 0.0).withUseIndex(true);
+ GDBTrainer trainer = new GDBBinaryClassifierOnTreesTrainer(0.3, 500, 3, 0.0)
+ .withUseIndex(true)
+ .withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.3));
Model<Vector, Double> mdl = fitter.apply(trainer, learningSample);
@@ -132,7 +141,7 @@ public class GDBTrainerTest {
double x = xs[j];
double y = ys[j];
double p = mdl.apply(VectorUtils.of(x));
- if(p != y)
+ if (p != y)
errorsCnt++;
}
@@ -142,7 +151,61 @@ public class GDBTrainerTest {
ModelsComposition composition = (ModelsComposition)mdl;
composition.getModels().forEach(m -> assertTrue(m instanceof DecisionTreeConditionalNode));
- assertEquals(500, composition.getModels().size());
+ assertTrue(composition.getModels().size() < 500);
assertTrue(composition.getPredictionsAggregator() instanceof WeightedPredictionsAggregator);
+
+ trainer = trainer.withCheckConvergenceStgyFactory(new ConvergenceCheckerStubFactory());
+ assertEquals(500, ((ModelsComposition)fitter.apply(trainer, learningSample)).getModels().size());
+ }
+
+ /** */
+ @Test
+ public void testUpdate() {
+ int sampleSize = 100;
+ double[] xs = new double[sampleSize];
+ double[] ys = new double[sampleSize];
+
+ for (int i = 0; i < sampleSize; i++) {
+ xs[i] = i;
+ ys[i] = ((int)(xs[i] / 10.0) % 2) == 0 ? -1.0 : 1.0;
+ }
+
+ Map<Integer, double[]> learningSample = new HashMap<>();
+ for (int i = 0; i < sampleSize; i++)
+ learningSample.put(i, new double[] {xs[i], ys[i]});
+ IgniteBiFunction<Integer, double[], Vector> fExtr = (k, v) -> VectorUtils.of(v[0]);
+ IgniteBiFunction<Integer, double[], Double> lExtr = (k, v) -> v[1];
+
+ GDBTrainer classifTrainer = new GDBBinaryClassifierOnTreesTrainer(0.3, 500, 3, 0.0)
+ .withUseIndex(true)
+ .withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.3));
+ GDBTrainer regressTrainer = new GDBRegressionOnTreesTrainer(0.3, 500, 3, 0.0)
+ .withUseIndex(true)
+ .withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.3));
+
+ testUpdate(learningSample, fExtr, lExtr, classifTrainer);
+ testUpdate(learningSample, fExtr, lExtr, regressTrainer);
+ }
+
+ /** */
+ private void testUpdate(Map<Integer, double[]> dataset, IgniteBiFunction<Integer, double[], Vector> fExtr,
+ IgniteBiFunction<Integer, double[], Double> lExtr, GDBTrainer trainer) {
+
+ ModelsComposition originalMdl = trainer.fit(dataset, 1, fExtr, lExtr);
+ ModelsComposition updatedOnSameDataset = trainer.update(originalMdl, dataset, 1, fExtr, lExtr);
+
+ LocalDatasetBuilder<Integer, double[]> epmtyDataset = new LocalDatasetBuilder<>(new HashMap<>(), 1);
+ ModelsComposition updatedOnEmptyDataset = trainer.updateModel(originalMdl, epmtyDataset, fExtr, lExtr);
+
+ dataset.forEach((k,v) -> {
+ Vector features = fExtr.apply(k, v);
+
+ Double originalAnswer = originalMdl.apply(features);
+ Double updatedMdlAnswer1 = updatedOnSameDataset.apply(features);
+ Double updatedMdlAnswer2 = updatedOnEmptyDataset.apply(features);
+
+ assertEquals(originalAnswer, updatedMdlAnswer1, 0.01);
+ assertEquals(originalAnswer, updatedMdlAnswer2, 0.01);
+ });
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceCheckerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceCheckerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceCheckerTest.java
new file mode 100644
index 0000000..50fdf8b
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceCheckerTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.convergence;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.ml.composition.ModelsComposition;
+import org.apache.ignite.ml.composition.boosting.loss.Loss;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
+import org.junit.Before;
+
+/** */
+public abstract class ConvergenceCheckerTest {
+ /** Not converged model. */
+ protected ModelsComposition notConvergedMdl = new ModelsComposition(Collections.emptyList(), null) {
+ @Override public Double apply(Vector features) {
+ return 2.1 * features.get(0);
+ }
+ };
+
+ /** Converged model. */
+ protected ModelsComposition convergedMdl = new ModelsComposition(Collections.emptyList(), null) {
+ @Override public Double apply(Vector features) {
+ return 2 * (features.get(0) + 1);
+ }
+ };
+
+ /** Features extractor. */
+ protected IgniteBiFunction<double[], Double, Vector> fExtr = (x, y) -> VectorUtils.of(x);
+
+ /** Label extractor. */
+ protected IgniteBiFunction<double[], Double, Double> lbExtr = (x, y) -> y;
+
+ /** Data. */
+ protected Map<double[], Double> data;
+
+ /** */
+ @Before
+ public void setUp() throws Exception {
+ data = new HashMap<>();
+ for(int i = 0; i < 10; i ++)
+ data.put(new double[]{i, i + 1}, (double)(2 * (i + 1)));
+ }
+
+ /** */
+ public ConvergenceChecker<double[], Double> createChecker(ConvergenceCheckerFactory factory,
+ LocalDatasetBuilder<double[], Double> datasetBuilder) {
+
+ return factory.create(data.size(),
+ x -> x,
+ new Loss() {
+ @Override public double error(long sampleSize, double lb, double mdlAnswer) {
+ return mdlAnswer - lb;
+ }
+
+ @Override public double gradient(long sampleSize, double lb, double mdlAnswer) {
+ return mdlAnswer - lb;
+ }
+ },
+ datasetBuilder, fExtr, lbExtr
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceCheckerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceCheckerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceCheckerTest.java
new file mode 100644
index 0000000..0b42db8
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceCheckerTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.convergence.mean;
+
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceChecker;
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceCheckerTest;
+import org.apache.ignite.ml.dataset.impl.local.LocalDataset;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.dataset.primitive.FeatureMatrixWithLabelsOnHeapData;
+import org.apache.ignite.ml.dataset.primitive.FeatureMatrixWithLabelsOnHeapDataBuilder;
+import org.apache.ignite.ml.dataset.primitive.builder.context.EmptyContextBuilder;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+/** */
+public class MeanAbsValueConvergenceCheckerTest extends ConvergenceCheckerTest {
+ /** */
+ @Test
+ public void testConvergenceChecking() {
+ LocalDatasetBuilder<double[], Double> datasetBuilder = new LocalDatasetBuilder<>(data, 1);
+ ConvergenceChecker<double[], Double> checker = createChecker(
+ new MeanAbsValueConvergenceCheckerFactory(0.1), datasetBuilder);
+
+ double error = checker.computeError(VectorUtils.of(1, 2), 4.0, notConvergedMdl);
+ Assert.assertEquals(1.9, error, 0.01);
+ Assert.assertFalse(checker.isConverged(datasetBuilder, notConvergedMdl));
+ Assert.assertTrue(checker.isConverged(datasetBuilder, convergedMdl));
+
+ try(LocalDataset<EmptyContext, FeatureMatrixWithLabelsOnHeapData> dataset = datasetBuilder.build(
+ new EmptyContextBuilder<>(), new FeatureMatrixWithLabelsOnHeapDataBuilder<>(fExtr, lbExtr))) {
+
+ double onDSError = checker.computeMeanErrorOnDataset(dataset, notConvergedMdl);
+ Assert.assertEquals(1.55, onDSError, 0.01);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /** Mean error more sensitive to anomalies in data */
+ @Test
+ public void testConvergenceCheckingWithAnomaliesInData() {
+ data.put(new double[]{10, 11}, 100000.0);
+ LocalDatasetBuilder<double[], Double> datasetBuilder = new LocalDatasetBuilder<>(data, 1);
+ ConvergenceChecker<double[], Double> checker = createChecker(
+ new MeanAbsValueConvergenceCheckerFactory(0.1), datasetBuilder);
+
+ try(LocalDataset<EmptyContext, FeatureMatrixWithLabelsOnHeapData> dataset = datasetBuilder.build(
+ new EmptyContextBuilder<>(), new FeatureMatrixWithLabelsOnHeapDataBuilder<>(fExtr, lbExtr))) {
+
+ double onDSError = checker.computeMeanErrorOnDataset(dataset, notConvergedMdl);
+ Assert.assertEquals(9090.41, onDSError, 0.01);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceCheckerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceCheckerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceCheckerTest.java
new file mode 100644
index 0000000..d6880b4
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceCheckerTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.convergence.median;
+
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceChecker;
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceCheckerTest;
+import org.apache.ignite.ml.dataset.impl.local.LocalDataset;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.dataset.primitive.FeatureMatrixWithLabelsOnHeapData;
+import org.apache.ignite.ml.dataset.primitive.FeatureMatrixWithLabelsOnHeapDataBuilder;
+import org.apache.ignite.ml.dataset.primitive.builder.context.EmptyContextBuilder;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+/** */
+public class MedianOfMedianConvergenceCheckerTest extends ConvergenceCheckerTest {
+ /** */
+ @Test
+ public void testConvergenceChecking() {
+ data.put(new double[]{10, 11}, 100000.0);
+ LocalDatasetBuilder<double[], Double> datasetBuilder = new LocalDatasetBuilder<>(data, 1);
+
+ ConvergenceChecker<double[], Double> checker = createChecker(
+ new MedianOfMedianConvergenceCheckerFactory(0.1), datasetBuilder);
+
+ double error = checker.computeError(VectorUtils.of(1, 2), 4.0, notConvergedMdl);
+ Assert.assertEquals(1.9, error, 0.01);
+ Assert.assertFalse(checker.isConverged(datasetBuilder, notConvergedMdl));
+ Assert.assertTrue(checker.isConverged(datasetBuilder, convergedMdl));
+
+ try(LocalDataset<EmptyContext, FeatureMatrixWithLabelsOnHeapData> dataset = datasetBuilder.build(
+ new EmptyContextBuilder<>(), new FeatureMatrixWithLabelsOnHeapDataBuilder<>(fExtr, lbExtr))) {
+
+ double onDSError = checker.computeMeanErrorOnDataset(dataset, notConvergedMdl);
+ Assert.assertEquals(1.6, onDSError, 0.01);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java
index f88fd3e..b06fd67 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java
@@ -17,185 +17,44 @@
package org.apache.ignite.ml.environment;
-import java.util.Arrays;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.ScanQuery;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.composition.ModelsComposition;
import org.apache.ignite.ml.dataset.feature.FeatureMeta;
import org.apache.ignite.ml.environment.logging.ConsoleLogger;
import org.apache.ignite.ml.environment.logging.MLLogger;
+import org.apache.ignite.ml.environment.parallelism.DefaultParallelismStrategy;
import org.apache.ignite.ml.environment.parallelism.ParallelismStrategy;
-import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.tree.randomforest.RandomForestRegressionTrainer;
import org.apache.ignite.ml.tree.randomforest.data.FeaturesCountSelectionStrategies;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.thread.IgniteThread;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
/**
* Tests for {@link LearningEnvironment} that require to start the whole Ignite infrastructure. IMPL NOTE based on
* RandomForestRegressionExample example.
*/
-public class LearningEnvironmentTest extends GridCommonAbstractTest {
- /** Number of nodes in grid */
- private static final int NODE_COUNT = 1;
-
- /** Ignite instance. */
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- for (int i = 1; i <= NODE_COUNT; i++)
- startGrid(i);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() {
- stopAllGrids();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override protected void beforeTest() {
- /* Grid instance. */
- ignite = grid(NODE_COUNT);
- ignite.configuration().setPeerClassLoadingEnabled(true);
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- }
-
+public class LearningEnvironmentTest {
/** */
+ @Test
public void testBasic() throws InterruptedException {
- AtomicReference<Integer> actualAmount = new AtomicReference<>(null);
- AtomicReference<Double> actualMse = new AtomicReference<>(null);
- AtomicReference<Double> actualMae = new AtomicReference<>(null);
-
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- LearningEnvironmentTest.class.getSimpleName(), () -> {
- IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
-
- AtomicInteger idx = new AtomicInteger(0);
- RandomForestRegressionTrainer trainer = new RandomForestRegressionTrainer(
- IntStream.range(0, data[0].length - 1).mapToObj(
- x -> new FeatureMeta("", idx.getAndIncrement(), false)).collect(Collectors.toList())
- ).withCountOfTrees(101)
- .withFeaturesCountSelectionStrgy(FeaturesCountSelectionStrategies.ONE_THIRD)
- .withMaxDepth(4)
- .withMinImpurityDelta(0.)
- .withSubsampleSize(0.3)
- .withSeed(0);
-
- trainer.setEnvironment(LearningEnvironment.builder()
- .withParallelismStrategy(ParallelismStrategy.Type.ON_DEFAULT_POOL)
- .withLoggingFactory(ConsoleLogger.factory(MLLogger.VerboseLevel.LOW))
- .build()
- );
-
- ModelsComposition randomForest = trainer.fit(ignite, dataCache,
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
- (k, v) -> v[v.length - 1]
- );
-
- double mse = 0.0;
- double mae = 0.0;
- int totalAmount = 0;
-
- try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, double[]> observation : observations) {
- double difference = estimatePrediction(randomForest, observation);
-
- mse += Math.pow(difference, 2.0);
- mae += Math.abs(difference);
-
- totalAmount++;
- }
- }
-
- actualAmount.set(totalAmount);
-
- mse = mse / totalAmount;
- actualMse.set(mse);
-
- mae = mae / totalAmount;
- actualMae.set(mae);
- });
-
- igniteThread.start();
- igniteThread.join();
-
- assertEquals("Total amount", 23, (int)actualAmount.get());
- assertTrue("Mean squared error (MSE)", actualMse.get() > 0);
- assertTrue("Mean absolute error (MAE)", actualMae.get() > 0);
+ RandomForestRegressionTrainer trainer = new RandomForestRegressionTrainer(
+ IntStream.range(0, 0).mapToObj(
+ x -> new FeatureMeta("", 0, false)).collect(Collectors.toList())
+ ).withCountOfTrees(101)
+ .withFeaturesCountSelectionStrgy(FeaturesCountSelectionStrategies.ONE_THIRD)
+ .withMaxDepth(4)
+ .withMinImpurityDelta(0.)
+ .withSubsampleSize(0.3)
+ .withSeed(0);
+
+ LearningEnvironment environment = LearningEnvironment.builder()
+ .withParallelismStrategy(ParallelismStrategy.Type.ON_DEFAULT_POOL)
+ .withLoggingFactory(ConsoleLogger.factory(MLLogger.VerboseLevel.LOW))
+ .build();
+ trainer.setEnvironment(environment);
+ assertEquals(DefaultParallelismStrategy.class, environment.parallelismStrategy().getClass());
+ assertEquals(ConsoleLogger.class, environment.logger().getClass());
}
-
- /** */
- private double estimatePrediction(ModelsComposition randomForest, Cache.Entry<Integer, double[]> observation) {
- double[] val = observation.getValue();
- double[] inputs = Arrays.copyOfRange(val, 0, val.length - 1);
- double groundTruth = val[val.length - 1];
-
- double prediction = randomForest.apply(VectorUtils.of(inputs));
-
- return prediction - groundTruth;
- }
-
- /**
- * Fills cache with data and returns it.
- *
- * @param ignite Ignite instance.
- * @return Filled Ignite Cache.
- */
- private IgniteCache<Integer, double[]> getTestCache(Ignite ignite) {
- CacheConfiguration<Integer, double[]> cacheConfiguration = new CacheConfiguration<>();
- cacheConfiguration.setName(UUID.randomUUID().toString());
- cacheConfiguration.setAffinity(new RendezvousAffinityFunction(false, 10));
-
- IgniteCache<Integer, double[]> cache = ignite.createCache(cacheConfiguration);
-
- for (int i = 0; i < data.length; i++)
- cache.put(i, data[i]);
-
- return cache;
- }
-
- /**
- * Part of the Boston housing dataset.
- */
- private static final double[][] data = {
- {0.02731,0.00,7.070,0,0.4690,6.4210,78.90,4.9671,2,242.0,17.80,396.90,9.14,21.60},
- {0.02729,0.00,7.070,0,0.4690,7.1850,61.10,4.9671,2,242.0,17.80,392.83,4.03,34.70},
- {0.03237,0.00,2.180,0,0.4580,6.9980,45.80,6.0622,3,222.0,18.70,394.63,2.94,33.40},
- {0.06905,0.00,2.180,0,0.4580,7.1470,54.20,6.0622,3,222.0,18.70,396.90,5.33,36.20},
- {0.02985,0.00,2.180,0,0.4580,6.4300,58.70,6.0622,3,222.0,18.70,394.12,5.21,28.70},
- {0.08829,12.50,7.870,0,0.5240,6.0120,66.60,5.5605,5,311.0,15.20,395.60,12.43,22.90},
- {0.14455,12.50,7.870,0,0.5240,6.1720,96.10,5.9505,5,311.0,15.20,396.90,19.15,27.10},
- {0.21124,12.50,7.870,0,0.5240,5.6310,100.00,6.0821,5,311.0,15.20,386.63,29.93,16.50},
- {0.17004,12.50,7.870,0,0.5240,6.0040,85.90,6.5921,5,311.0,15.20,386.71,17.10,18.90},
- {0.22489,12.50,7.870,0,0.5240,6.3770,94.30,6.3467,5,311.0,15.20,392.52,20.45,15.00},
- {0.11747,12.50,7.870,0,0.5240,6.0090,82.90,6.2267,5,311.0,15.20,396.90,13.27,18.90},
- {0.09378,12.50,7.870,0,0.5240,5.8890,39.00,5.4509,5,311.0,15.20,390.50,15.71,21.70},
- {0.62976,0.00,8.140,0,0.5380,5.9490,61.80,4.7075,4,307.0,21.00,396.90,8.26,20.40},
- {0.63796,0.00,8.140,0,0.5380,6.0960,84.50,4.4619,4,307.0,21.00,380.02,10.26,18.20},
- {0.62739,0.00,8.140,0,0.5380,5.8340,56.50,4.4986,4,307.0,21.00,395.62,8.47,19.90},
- {1.05393,0.00,8.140,0,0.5380,5.9350,29.30,4.4986,4,307.0,21.00,386.85,6.58,23.10},
- {0.78420,0.00,8.140,0,0.5380,5.9900,81.70,4.2579,4,307.0,21.00,386.75,14.67,17.50},
- {0.80271,0.00,8.140,0,0.5380,5.4560,36.60,3.7965,4,307.0,21.00,288.99,11.69,20.20},
- {0.72580,0.00,8.140,0,0.5380,5.7270,69.50,3.7965,4,307.0,21.00,390.95,11.28,18.20},
- {1.25179,0.00,8.140,0,0.5380,5.5700,98.10,3.7979,4,307.0,21.00,376.57,21.02,13.60},
- {0.85204,0.00,8.140,0,0.5380,5.9650,89.20,4.0123,4,307.0,21.00,392.53,13.83,19.60},
- {1.23247,0.00,8.140,0,0.5380,6.1420,91.70,3.9769,4,307.0,21.00,396.90,18.72,15.20},
- {0.98843,0.00,8.140,0,0.5380,5.8130,100.00,4.0952,4,307.0,21.00,394.54,19.88,14.50}
- };
-
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/test/java/org/apache/ignite/ml/knn/ANNClassificationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/ANNClassificationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/ANNClassificationTest.java
index d8fb620..199644b 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/ANNClassificationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/ANNClassificationTest.java
@@ -93,17 +93,21 @@ public class ANNClassificationTest extends TrainerTest {
.withDistanceMeasure(new EuclideanDistance())
.withStrategy(NNStrategy.SIMPLE);
- ANNClassificationModel updatedOnSameDataset = trainer.withSeed(1234L).update(originalMdl,
+ ANNClassificationModel updatedOnSameDataset = (ANNClassificationModel) trainer.withSeed(1234L).update(originalMdl,
cacheMock, parts,
(k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
(k, v) -> v[2]
- );
+ ).withK(3)
+ .withDistanceMeasure(new EuclideanDistance())
+ .withStrategy(NNStrategy.SIMPLE);
- ANNClassificationModel updatedOnEmptyDataset = trainer.withSeed(1234L).update(originalMdl,
+ ANNClassificationModel updatedOnEmptyDataset = (ANNClassificationModel) trainer.withSeed(1234L).update(originalMdl,
new HashMap<Integer, double[]>(), parts,
(k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
(k, v) -> v[2]
- );
+ ).withK(3)
+ .withDistanceMeasure(new EuclideanDistance())
+ .withStrategy(NNStrategy.SIMPLE);
Vector v1 = VectorUtils.of(550, 550);
Vector v2 = VectorUtils.of(-550, -550);
[20/50] [abbrv] ignite git commit: IGNITE-8829 Annotated
configuration properties on TcpCommunicationSpi that lack required
annotations - Fixes #4583.
Posted by ag...@apache.org.
IGNITE-8829 Annotated configuration properties on TcpCommunicationSpi that lack required annotations - Fixes #4583.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/625e2610
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/625e2610
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/625e2610
Branch: refs/heads/ignite-5960
Commit: 625e261078f1df215f51d17a0e84daa847f86bef
Parents: de30a86
Author: NSAmelchev <ns...@gmail.com>
Authored: Tue Sep 4 15:08:38 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Sep 4 15:08:38 2018 +0300
----------------------------------------------------------------------
.../communication/tcp/TcpCommunicationSpi.java | 71 +++++++++++++-------
1 file changed, 45 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/625e2610/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 77e85c1..4ab1dd4 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -204,16 +204,22 @@ import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastRecei
* <h2 class="header">Optional</h2>
* The following configuration parameters are optional:
* <ul>
+ * <li>Address resolver (see {@link #setAddressResolver(AddressResolver)}</li>
* <li>Node local IP address (see {@link #setLocalAddress(String)})</li>
* <li>Node local port number (see {@link #setLocalPort(int)})</li>
* <li>Local port range (see {@link #setLocalPortRange(int)}</li>
+ * <li>Use paired connections (see {@link #setUsePairedConnections(boolean)}</li>
* <li>Connections per node (see {@link #setConnectionsPerNode(int)})</li>
+ * <li>Shared memory port (see {@link #setSharedMemoryPort(int)}</li>
* <li>Idle connection timeout (see {@link #setIdleConnectionTimeout(long)})</li>
* <li>Direct or heap buffer allocation (see {@link #setDirectBuffer(boolean)})</li>
* <li>Direct or heap buffer allocation for sending (see {@link #setDirectSendBuffer(boolean)})</li>
* <li>Count of selectors and selector threads for NIO server (see {@link #setSelectorsCount(int)})</li>
+ * <li>Selector thread busy-loop iterations (see {@link #setSelectorSpins(long)}</li>
* <li>{@code TCP_NODELAY} socket option for sockets (see {@link #setTcpNoDelay(boolean)})</li>
+ * <li>Filter reachable addresses (see {@link #setFilterReachableAddresses(boolean)} </li>
* <li>Message queue limit (see {@link #setMessageQueueLimit(int)})</li>
+ * <li>Slow client queue limit (see {@link #setSlowClientQueueLimit(int)})</li>
* <li>Connect timeout (see {@link #setConnectTimeout(long)})</li>
* <li>Maximum connect timeout (see {@link #setMaxConnectTimeout(long)})</li>
* <li>Reconnect attempts count (see {@link #setReconnectCount(int)})</li>
@@ -382,7 +388,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
private final GridNioServerListener<Message> srvLsnr =
new GridNioServerListenerAdapter<Message>() {
@Override public void onSessionWriteTimeout(GridNioSession ses) {
- LT.warn(log,"Communication SPI session write timed out (consider increasing " +
+ LT.warn(log, "Communication SPI session write timed out (consider increasing " +
"'socketWriteTimeout' " + "configuration property) [remoteAddr=" + ses.remoteAddress() +
", writeTimeout=" + sockWriteTimeout + ']');
@@ -499,7 +505,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
boolean unknownNode = true;
if (discoverySpi instanceof TcpDiscoverySpi) {
- TcpDiscoverySpi tcpDiscoverySpi = (TcpDiscoverySpi) discoverySpi;
+ TcpDiscoverySpi tcpDiscoverySpi = (TcpDiscoverySpi)discoverySpi;
ClusterNode node0 = tcpDiscoverySpi.getNode0(sndId);
@@ -511,7 +517,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
}
else if (discoverySpi instanceof IgniteDiscoverySpi)
- unknownNode = !((IgniteDiscoverySpi) discoverySpi).knownNode(sndId);
+ unknownNode = !((IgniteDiscoverySpi)discoverySpi).knownNode(sndId);
if (unknownNode) {
U.warn(log, "Close incoming connection, unknown node [nodeId=" + sndId + ", ses=" + ses + ']');
@@ -636,7 +642,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
if (log.isDebugEnabled())
log.debug("Received incoming connection from remote node " +
- "[rmtNode=" + rmtNode.id() + ", reserved=" + reserved +
+ "[rmtNode=" + rmtNode.id() + ", reserved=" + reserved +
", recovery=" + recoveryDesc + ']');
if (reserved) {
@@ -655,9 +661,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
if (oldFut instanceof ConnectFuture && locNode.order() < rmtNode.order()) {
if (log.isInfoEnabled()) {
log.info("Received incoming connection from remote node while " +
- "connecting to this node, rejecting [locNode=" + locNode.id() +
- ", locNodeOrder=" + locNode.order() + ", rmtNode=" + rmtNode.id() +
- ", rmtNodeOrder=" + rmtNode.order() + ']');
+ "connecting to this node, rejecting [locNode=" + locNode.id() +
+ ", locNodeOrder=" + locNode.order() + ", rmtNode=" + rmtNode.id() +
+ ", rmtNodeOrder=" + rmtNode.order() + ']');
}
ses.send(new RecoveryLastReceivedMessage(ALREADY_CONNECTED));
@@ -928,7 +934,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
catch (IgniteCheckedException e) {
if (log.isDebugEnabled())
log.debug("Failed to send recovery handshake " +
- "[rmtNode=" + rmtNode.id() + ", err=" + e + ']');
+ "[rmtNode=" + rmtNode.id() + ", err=" + e + ']');
recoveryDesc.release();
}
@@ -1010,14 +1016,14 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
msgFut.get();
GridTcpNioCommunicationClient client =
- connected(recoveryDesc, ses, rmtNode, msg.received(), false, createClient);
+ connected(recoveryDesc, ses, rmtNode, msg.received(), false, createClient);
fut.onDone(client);
}
catch (IgniteCheckedException e) {
if (log.isDebugEnabled())
log.debug("Failed to send recovery handshake " +
- "[rmtNode=" + rmtNode.id() + ", err=" + e + ']');
+ "[rmtNode=" + rmtNode.id() + ", err=" + e + ']');
recoveryDesc.release();
@@ -1210,6 +1216,15 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
/**
+ * See {@link #setAddressResolver(AddressResolver)}.
+ *
+ * @return Address resolver.
+ */
+ public AddressResolver getAddressResolver() {
+ return addrRslvr;
+ }
+
+ /**
* Injects resources.
*
* @param ignite Ignite.
@@ -1330,9 +1345,10 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
* Default is {@code false}.
*
* @param usePairedConnections {@code true} to use paired connections and {@code false} otherwise.
- * @see #getConnectionsPerNode()
* @return {@code this} for chaining.
+ * @see #getConnectionsPerNode()
*/
+ @IgniteSpiConfiguration(optional = true)
public TcpCommunicationSpi setUsePairedConnections(boolean usePairedConnections) {
this.usePairedConnections = usePairedConnections;
@@ -1345,9 +1361,10 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
* half for outgoing messages.
*
* @param maxConnectionsPerNode Number of connections per node.
- * @see #isUsePairedConnections()
* @return {@code this} for chaining.
+ * @see #isUsePairedConnections()
*/
+ @IgniteSpiConfiguration(optional = true)
public TcpCommunicationSpi setConnectionsPerNode(int maxConnectionsPerNode) {
this.connectionsPerNode = maxConnectionsPerNode;
@@ -1357,7 +1374,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
/**
* See {@link #setConnectionsPerNode(int)}.
*
- * @return Number of connections per node.
+ * @return Number of connections per node.
*/
public int getConnectionsPerNode() {
return connectionsPerNode;
@@ -1513,7 +1530,8 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
* See {@link #setConnectTimeout(long)}.
*
* @return Connect timeout.
- */public long getConnectTimeout() {
+ */
+ public long getConnectTimeout() {
return connTimeout;
}
@@ -1671,6 +1689,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
* @param selectorSpins Selector thread busy-loop iterations.
* @return {@code this} for chaining.
*/
+ @IgniteSpiConfiguration(optional = true)
public TcpCommunicationSpi setSelectorSpins(long selectorSpins) {
this.selectorSpins = selectorSpins;
@@ -1832,6 +1851,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
* @param slowClientQueueLimit Slow client queue limit.
* @return {@code this} for chaining.
*/
+ @IgniteSpiConfiguration(optional = true)
public TcpCommunicationSpi setSlowClientQueueLimit(int slowClientQueueLimit) {
this.slowClientQueueLimit = slowClientQueueLimit;
@@ -2223,7 +2243,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
log.debug(startInfo());
}
- /** {@inheritDoc} }*/
+ /** {@inheritDoc} } */
@Override public void onContextInitialized0(IgniteSpiContext spiCtx) throws IgniteSpiException {
spiCtx.registerPort(boundTcpPort, IgnitePortProtocol.TCP);
@@ -2331,12 +2351,12 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
IgniteBiInClosure<GridNioSession, Integer> queueSizeMonitor =
!clientMode && slowClientQueueLimit > 0 ?
- new CI2<GridNioSession, Integer>() {
- @Override public void apply(GridNioSession ses, Integer qSize) {
- checkClientQueueSize(ses, qSize);
- }
- } :
- null;
+ new CI2<GridNioSession, Integer>() {
+ @Override public void apply(GridNioSession ses, Integer qSize) {
+ checkClientQueueSize(ses, qSize);
+ }
+ } :
+ null;
GridNioFilter[] filters;
@@ -3262,7 +3282,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
GridNioSession ses = recoveryDesc.session();
if (ses != null) {
- while(ses.closeTime() == 0)
+ while (ses.closeTime() == 0)
ses.close();
}
@@ -3485,7 +3505,6 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
* @param node Remote node.
* @param addrs Remote node addresses.
* @param errs TCP client creation errors.
- *
* @throws IgniteCheckedException If failed.
*/
protected void processClientCreationError(
@@ -3592,8 +3611,8 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
* @param timeout Timeout for handshake.
* @param sslMeta Session meta.
* @param handshakeConnIdx Non null connection index if need send it in handshake.
- * @throws IgniteCheckedException If handshake failed or wasn't completed withing timeout.
* @return Handshake response.
+ * @throws IgniteCheckedException If handshake failed or wasn't completed withing timeout.
*/
@SuppressWarnings("ThrowFromFinallyBlock")
private long safeTcpHandshake(
@@ -4113,7 +4132,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
/** {@inheritDoc} */
@Override public void onEvent(Event evt) {
assert evt instanceof DiscoveryEvent : evt;
- assert evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED ;
+ assert evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED;
onNodeLeft(((DiscoveryEvent)evt).eventNode().id());
}
@@ -4194,7 +4213,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
srvLsnr,
writerFactory,
new GridNioCodecFilter(
- new GridDirectParser(log.getLogger(GridDirectParser.class),msgFactory, readerFactory),
+ new GridDirectParser(log.getLogger(GridDirectParser.class), msgFactory, readerFactory),
log,
true),
new GridConnectionBytesVerifyFilter(log)
[04/50] [abbrv] ignite git commit: IGNITE-8189 Improved
ZkDistributedCollectDataFuture#deleteFutureData implementation - Fixes #4537.
Posted by ag...@apache.org.
IGNITE-8189 Improved ZkDistributedCollectDataFuture#deleteFutureData implementation - Fixes #4537.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2cccfae4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2cccfae4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2cccfae4
Branch: refs/heads/ignite-5960
Commit: 2cccfae49106f36a4f0b2b52a81889a295f52677
Parents: d1b3882
Author: NSAmelchev <ns...@gmail.com>
Authored: Fri Aug 31 15:47:16 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Aug 31 15:47:16 2018 +0300
----------------------------------------------------------------------
.../ZkDistributedCollectDataFuture.java | 19 ++++------
.../discovery/zk/internal/ZookeeperClient.java | 40 ++++++++------------
.../zk/internal/ZookeeperDiscoveryImpl.java | 39 +++++++------------
.../zk/internal/ZookeeperClientTest.java | 13 ++-----
4 files changed, 40 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2cccfae4/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
index e9b28e1..e710055 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
@@ -18,6 +18,7 @@
package org.apache.ignite.spi.discovery.zk.internal;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.UUID;
@@ -145,23 +146,19 @@ class ZkDistributedCollectDataFuture extends GridFutureAdapter<Void> {
UUID futId,
IgniteLogger log
) throws Exception {
- // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8189
+ List<String> batch = new LinkedList<>();
+
String evtDir = paths.distributedFutureBasePath(futId);
- try {
- client.deleteAll(evtDir,
- client.getChildrenIfPathExists(evtDir),
- -1);
- }
- catch (KeeperException.NoNodeException e) {
- U.log(log, "Node for deletion was not found: " + e.getPath());
+ if (client.exists(evtDir)) {
+ batch.addAll(client.getChildrenPaths(evtDir));
- // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8189
+ batch.add(evtDir);
}
- client.deleteIfExists(evtDir, -1);
+ batch.add(paths.distributedFutureResultPath(futId));
- client.deleteIfExists(paths.distributedFutureResultPath(futId), -1);
+ client.deleteAll(batch, -1);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/2cccfae4/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
index b58f0ce..39417c2 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
@@ -504,9 +504,7 @@ public class ZookeeperClient implements Watcher {
* @throws ZookeeperClientFailedException If connection to zk was lost.
* @throws InterruptedException If interrupted.
*/
- List<String> getChildren(String path)
- throws ZookeeperClientFailedException, InterruptedException
- {
+ List<String> getChildren(String path) throws ZookeeperClientFailedException, InterruptedException {
for (;;) {
long connStartTime = this.connStartTime;
@@ -520,29 +518,23 @@ public class ZookeeperClient implements Watcher {
}
/**
+ * Get children paths.
+ *
* @param path Path.
- * @return Children nodes.
- * @throws KeeperException.NoNodeException If provided path does not exist.
+ * @return Children paths.
* @throws ZookeeperClientFailedException If connection to zk was lost.
* @throws InterruptedException If interrupted.
*/
- List<String> getChildrenIfPathExists(String path) throws
- KeeperException.NoNodeException, InterruptedException, ZookeeperClientFailedException {
- for (;;) {
- long connStartTime = this.connStartTime;
+ List<String> getChildrenPaths(String path) throws ZookeeperClientFailedException, InterruptedException {
+ List<String> children = getChildren(path);
- try {
- return zk.getChildren(path, false);
- }
- catch (KeeperException.NoNodeException e) {
- throw e;
- }
- catch (Exception e) {
- onZookeeperError(connStartTime, e);
- }
- }
- }
+ ArrayList<String> paths = new ArrayList(children.size());
+ for (String child : children)
+ paths.add(path + "/" + child);
+
+ return paths;
+ }
/**
* @param path Path.
@@ -593,7 +585,7 @@ public class ZookeeperClient implements Watcher {
* @throws ZookeeperClientFailedException If connection to zk was lost.
* @throws InterruptedException If interrupted.
*/
- void deleteAll(@Nullable String parent, List<String> paths, int ver)
+ void deleteAll(List<String> paths, int ver)
throws ZookeeperClientFailedException, InterruptedException {
if (paths.isEmpty())
return;
@@ -605,10 +597,8 @@ public class ZookeeperClient implements Watcher {
List<Op> batch = new LinkedList<>();
for (String path : paths) {
- String path0 = parent != null ? parent + "/" + path : path;
-
//TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8187
- int size = requestOverhead(path0) + 17 /* overhead */;
+ int size = requestOverhead(path) + 17 /* overhead */;
assert size <= MAX_REQ_SIZE;
@@ -620,7 +610,7 @@ public class ZookeeperClient implements Watcher {
batchSize = 0;
}
- batch.add(Op.delete(path0, ver));
+ batch.add(Op.delete(path, ver));
batchSize += size;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/2cccfae4/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 4579efd..069b3e2 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -27,6 +27,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -2263,33 +2264,27 @@ public class ZookeeperDiscoveryImpl {
ZookeeperClient client = rtState.zkClient;
- // TODO ZK: use multi, better batching + max-size safe + NoNodeException safe.
- List<String> evtChildren = rtState.zkClient.getChildren(zkPaths.evtsPath);
+ List<String> batch = new LinkedList<>();
- for (String evtPath : evtChildren) {
- String evtDir = zkPaths.evtsPath + "/" + evtPath;
+ List<String> evtChildren = client.getChildrenPaths(zkPaths.evtsPath);
- removeChildren(evtDir);
- }
+ for (String evtPath : evtChildren)
+ batch.addAll(client.getChildrenPaths(evtPath));
+
+ batch.addAll(evtChildren);
- client.deleteAll(zkPaths.evtsPath, evtChildren, -1);
+ batch.addAll(client.getChildrenPaths(zkPaths.customEvtsDir));
- client.deleteAll(zkPaths.customEvtsDir,
- client.getChildren(zkPaths.customEvtsDir),
- -1);
+ batch.addAll(client.getChildrenPaths(zkPaths.customEvtsPartsDir));
- rtState.zkClient.deleteAll(zkPaths.customEvtsPartsDir,
- rtState.zkClient.getChildren(zkPaths.customEvtsPartsDir),
- -1);
+ batch.addAll(client.getChildrenPaths(zkPaths.customEvtsAcksDir));
- rtState.zkClient.deleteAll(zkPaths.customEvtsAcksDir,
- rtState.zkClient.getChildren(zkPaths.customEvtsAcksDir),
- -1);
+ client.deleteAll(batch, -1);
if (startInternalOrder > 0) {
- for (String alive : rtState.zkClient.getChildren(zkPaths.aliveNodesDir)) {
+ for (String alive : client.getChildren(zkPaths.aliveNodesDir)) {
if (ZkIgnitePaths.aliveInternalId(alive) < startInternalOrder)
- rtState.zkClient.deleteIfExists(zkPaths.aliveNodesDir + "/" + alive, -1);
+ client.deleteIfExists(zkPaths.aliveNodesDir + "/" + alive, -1);
}
}
@@ -2302,14 +2297,6 @@ public class ZookeeperDiscoveryImpl {
}
/**
- * @param path Path.
- * @throws Exception If failed.
- */
- private void removeChildren(String path) throws Exception {
- rtState.zkClient.deleteAll(path, rtState.zkClient.getChildren(path), -1);
- }
-
- /**
* @param zkClient Client.
* @param evtPath Event path.
* @param sndNodeId Sender node ID.
http://git-wip-us.apache.org/repos/asf/ignite/blob/2cccfae4/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
index 0d64980..7c9ec51 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
@@ -196,12 +196,12 @@ public class ZookeeperClientTest extends GridCommonAbstractTest {
client.createIfNeeded("/apacheIgnite/1", null, CreateMode.PERSISTENT);
client.createIfNeeded("/apacheIgnite/2", null, CreateMode.PERSISTENT);
- client.deleteAll("/apacheIgnite", Arrays.asList("1", "2"), -1);
+ client.deleteAll(Arrays.asList("/apacheIgnite/1", "/apacheIgnite/2"), -1);
assertTrue(client.getChildren("/apacheIgnite").isEmpty());
client.createIfNeeded("/apacheIgnite/1", null, CreateMode.PERSISTENT);
- client.deleteAll("/apacheIgnite", Collections.singletonList("1"), -1);
+ client.deleteAll(Collections.singletonList("/apacheIgnite/1"), -1);
assertTrue(client.getChildren("/apacheIgnite").isEmpty());
}
@@ -227,12 +227,7 @@ public class ZookeeperClientTest extends GridCommonAbstractTest {
assertEquals(cnt, client.getChildren("/apacheIgnite").size());
- List<String> subPaths = new ArrayList<>(cnt);
-
- for (int i = 0; i < cnt; i++)
- subPaths.add(String.valueOf(i));
-
- client.deleteAll("/apacheIgnite", subPaths, -1);
+ client.deleteAll(paths, -1);
assertTrue(client.getChildren("/apacheIgnite").isEmpty());
}
@@ -249,7 +244,7 @@ public class ZookeeperClientTest extends GridCommonAbstractTest {
client.createIfNeeded("/apacheIgnite/1", null, CreateMode.PERSISTENT);
client.createIfNeeded("/apacheIgnite/2", null, CreateMode.PERSISTENT);
- client.deleteAll("/apacheIgnite", Arrays.asList("1", "2", "3"), -1);
+ client.deleteAll(Arrays.asList("/apacheIgnite/1", "/apacheIgnite/2", "/apacheIgnite/3"), -1);
assertTrue(client.getChildren("/apacheIgnite").isEmpty());
}
[30/50] [abbrv] ignite git commit: IGNITE-6454 muted flaky tests -
Fixes #4682.
Posted by ag...@apache.org.
IGNITE-6454 muted flaky tests - Fixes #4682.
Signed-off-by: Dmitriy Govorukhin <dm...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9862dbe0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9862dbe0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9862dbe0
Branch: refs/heads/ignite-5960
Commit: 9862dbe03b733523cd3ead8a8c94bd800afe40c4
Parents: 5b4d563
Author: dgarus <ga...@gmail.com>
Authored: Wed Sep 5 11:29:01 2018 +0300
Committer: Dmitriy Govorukhin <dm...@gmail.com>
Committed: Wed Sep 5 11:29:01 2018 +0300
----------------------------------------------------------------------
...achePartitionedDataStructuresFailoverSelfTest.java | 14 ++++++++++++++
...CacheReplicatedDataStructuresFailoverSelfTest.java | 14 ++++++++++++++
2 files changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9862dbe0/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedDataStructuresFailoverSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedDataStructuresFailoverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedDataStructuresFailoverSelfTest.java
index ecb2df9..eecfefe 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedDataStructuresFailoverSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/partitioned/GridCachePartitionedDataStructuresFailoverSelfTest.java
@@ -38,4 +38,18 @@ public class GridCachePartitionedDataStructuresFailoverSelfTest
@Override protected CacheAtomicityMode collectionCacheAtomicityMode() {
return TRANSACTIONAL;
}
+
+ /**
+ *
+ */
+ @Override public void testReentrantLockConstantTopologyChangeNonFailoverSafe() {
+ fail("https://issues.apache.org/jira/browse/IGNITE-6454");
+ }
+
+ /**
+ *
+ */
+ @Override public void testFairReentrantLockConstantTopologyChangeNonFailoverSafe() {
+ fail("https://issues.apache.org/jira/browse/IGNITE-6454");
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9862dbe0/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/replicated/GridCacheReplicatedDataStructuresFailoverSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/replicated/GridCacheReplicatedDataStructuresFailoverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/replicated/GridCacheReplicatedDataStructuresFailoverSelfTest.java
index cb6e77e..27fbdcf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/replicated/GridCacheReplicatedDataStructuresFailoverSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/replicated/GridCacheReplicatedDataStructuresFailoverSelfTest.java
@@ -52,4 +52,18 @@ public class GridCacheReplicatedDataStructuresFailoverSelfTest
@Override public void testReentrantLockConstantMultipleTopologyChangeNonFailoverSafe() {
fail("https://issues.apache.org/jira/browse/IGNITE-6454");
}
+
+ /**
+ *
+ */
+ @Override public void testReentrantLockConstantTopologyChangeNonFailoverSafe() {
+ fail("https://issues.apache.org/jira/browse/IGNITE-6454");
+ }
+
+ /**
+ *
+ */
+ @Override public void testFairReentrantLockConstantTopologyChangeNonFailoverSafe() {
+ fail("https://issues.apache.org/jira/browse/IGNITE-6454");
+ }
}
[46/50] [abbrv] ignite git commit: IGNITE-9482: [ML] Refactor all
trainers' settters to withFieldName format for meta-algorithms
Posted by ag...@apache.org.
IGNITE-9482: [ML] Refactor all trainers' settters to
withFieldName format for meta-algorithms
this closes #4699
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b10ba044
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b10ba044
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b10ba044
Branch: refs/heads/ignite-5960
Commit: b10ba044d2d4357738c70f924b94e2e6a50c5f20
Parents: 0a23658
Author: zaleslaw <za...@gmail.com>
Authored: Fri Sep 7 13:31:58 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Fri Sep 7 13:31:58 2018 +0300
----------------------------------------------------------------------
.../clustering/KMeansClusterizationExample.java | 4 +-
.../RandomForestClassificationExample.java | 2 +-
.../RandomForestRegressionExample.java | 2 +-
.../clustering/kmeans/ClusterizationModel.java | 4 +-
.../ml/clustering/kmeans/KMeansModel.java | 4 +-
.../ml/clustering/kmeans/KMeansTrainer.java | 8 +-
.../ml/knn/ann/ANNClassificationTrainer.java | 4 +-
.../org/apache/ignite/ml/nn/MLPTrainer.java | 165 ++++++++++++++++++-
.../binarization/BinarizationPreprocessor.java | 4 +-
.../binarization/BinarizationTrainer.java | 9 +-
.../preprocessing/encoding/EncoderTrainer.java | 11 ++
.../linear/LinearRegressionSGDTrainer.java | 58 ++++++-
.../binomial/LogisticRegressionSGDTrainer.java | 107 +++++++++++-
.../LogRegressionMultiClassTrainer.java | 26 +--
.../ignite/ml/selection/cv/CrossValidation.java | 2 +-
.../ml/selection/paramgrid/ParamGrid.java | 14 +-
.../SVMLinearBinaryClassificationTrainer.java | 22 +--
...VMLinearMultiClassClassificationTrainer.java | 22 +--
.../org/apache/ignite/ml/tree/DecisionTree.java | 4 +-
.../tree/DecisionTreeClassificationTrainer.java | 4 +-
.../ml/tree/DecisionTreeRegressionTrainer.java | 10 +-
.../GDBBinaryClassifierOnTreesTrainer.java | 73 ++++++--
.../boosting/GDBRegressionOnTreesTrainer.java | 67 +++++++-
.../tree/randomforest/RandomForestTrainer.java | 14 +-
.../ignite/ml/clustering/KMeansModelTest.java | 4 +-
.../ignite/ml/clustering/KMeansTrainerTest.java | 8 +-
.../ignite/ml/common/LocalModelsTest.java | 2 +-
.../ml/composition/boosting/GDBTrainerTest.java | 8 +-
.../ml/environment/LearningEnvironmentTest.java | 2 +-
.../binarization/BinarizationTrainerTest.java | 6 +-
.../logistic/LogRegMultiClassTrainerTest.java | 8 +-
.../tree/DecisionTreeRegressionTrainerTest.java | 2 +-
.../ml/tree/randomforest/RandomForestTest.java | 2 +-
33 files changed, 540 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
index b96cbce..152375a 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
@@ -70,8 +70,8 @@ public class KMeansClusterizationExample {
);
System.out.println(">>> KMeans centroids");
- Tracer.showAscii(mdl.centers()[0]);
- Tracer.showAscii(mdl.centers()[1]);
+ Tracer.showAscii(mdl.getCenters()[0]);
+ Tracer.showAscii(mdl.getCenters()[1]);
System.out.println(">>>");
System.out.println(">>> -----------------------------------");
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java
index aa13943..6194153 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java
@@ -74,7 +74,7 @@ public class RandomForestClassificationExample {
.withFeaturesCountSelectionStrgy(FeaturesCountSelectionStrategies.ONE_THIRD)
.withMaxDepth(4)
.withMinImpurityDelta(0.)
- .withSubsampleSize(0.3)
+ .withSubSampleSize(0.3)
.withSeed(0);
System.out.println(">>> Configured trainer: " + classifier.getClass().getSimpleName());
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java
index e2bfe8b..5f010f2 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java
@@ -78,7 +78,7 @@ public class RandomForestRegressionExample {
.withFeaturesCountSelectionStrgy(FeaturesCountSelectionStrategies.ONE_THIRD)
.withMaxDepth(4)
.withMinImpurityDelta(0.)
- .withSubsampleSize(0.3)
+ .withSubSampleSize(0.3)
.withSeed(0);
trainer.setEnvironment(LearningEnvironment.builder()
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/ClusterizationModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/ClusterizationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/ClusterizationModel.java
index 474a463..43e1899 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/ClusterizationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/ClusterizationModel.java
@@ -22,8 +22,8 @@ import org.apache.ignite.ml.Model;
/** Base interface for all clusterization models. */
public interface ClusterizationModel<P, V> extends Model<P, V> {
/** Gets the clusters count. */
- public int amountOfClusters();
+ public int getAmountOfClusters();
/** Get cluster centers. */
- public P[] centers();
+ public P[] getCenters();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModel.java
index bdfa1b6..e07f4f0 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModel.java
@@ -54,12 +54,12 @@ public class KMeansModel implements ClusterizationModel<Vector, Integer>, Export
}
/** Amount of centers in clusterization. */
- @Override public int amountOfClusters() {
+ @Override public int getAmountOfClusters() {
return centers.length;
}
/** Get centers of clusters. */
- @Override public Vector[] centers() {
+ @Override public Vector[] getCenters() {
return Arrays.copyOf(centers, centers.length);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
index 2596dbc..a20d5da 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
@@ -106,7 +106,7 @@ public class KMeansTrainer extends SingleLabelDatasetTrainer<KMeansModel> {
return getLastTrainedModelOrThrowEmptyDatasetException(mdl);
centers = Optional.ofNullable(mdl)
- .map(KMeansModel::centers)
+ .map(KMeansModel::getCenters)
.orElseGet(() -> initClusterCentersRandomly(dataset, k));
boolean converged = false;
@@ -143,7 +143,7 @@ public class KMeansTrainer extends SingleLabelDatasetTrainer<KMeansModel> {
/** {@inheritDoc} */
@Override protected boolean checkState(KMeansModel mdl) {
- return mdl.centers().length == k && mdl.distanceMeasure().equals(distance);
+ return mdl.getCenters().length == k && mdl.distanceMeasure().equals(distance);
}
/**
@@ -313,7 +313,7 @@ public class KMeansTrainer extends SingleLabelDatasetTrainer<KMeansModel> {
*
* @return The parameter value.
*/
- public int getK() {
+ public int getAmountOfClusters() {
return k;
}
@@ -323,7 +323,7 @@ public class KMeansTrainer extends SingleLabelDatasetTrainer<KMeansModel> {
* @param k The parameter value.
* @return Model with new amount of clusters parameter value.
*/
- public KMeansTrainer withK(int k) {
+ public KMeansTrainer withAmountOfClusters(int k) {
this.k = k;
return this;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java
index 3e32b67..e56a10a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java
@@ -131,7 +131,7 @@ public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClass
private <K, V> List<Vector> getCentroids(IgniteBiFunction<K, V, Vector> featureExtractor,
IgniteBiFunction<K, V, Double> lbExtractor, DatasetBuilder<K, V> datasetBuilder) {
KMeansTrainer trainer = new KMeansTrainer()
- .withK(k)
+ .withAmountOfClusters(k)
.withMaxIterations(maxIterations)
.withSeed(seed)
.withDistance(distance)
@@ -143,7 +143,7 @@ public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClass
lbExtractor
);
- return Arrays.asList(mdl.centers());
+ return Arrays.asList(mdl.getCenters());
}
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
index 8f1a4cb..1cac909 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
@@ -46,25 +46,25 @@ import org.apache.ignite.ml.util.Utils;
*/
public class MLPTrainer<P extends Serializable> extends MultiLabelDatasetTrainer<MultilayerPerceptron> {
/** Multilayer perceptron architecture supplier that defines layers and activators. */
- private final IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier;
+ private IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier;
/** Loss function to be minimized during the training. */
- private final IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
+ private IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
/** Update strategy that defines how to update model parameters during the training. */
- private final UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy;
+ private UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy;
/** Maximal number of iterations before the training will be stopped. */
- private final int maxIterations;
+ private int maxIterations = 100;
/** Batch size (per every partition). */
- private final int batchSize;
+ private int batchSize = 100;
/** Maximal number of local iterations before synchronization. */
- private final int locIterations;
+ private int locIterations = 100;
/** Multilayer perceptron model initializer. */
- private final long seed;
+ private long seed = 1234L;
/**
* Constructs a new instance of multilayer perceptron trainer.
@@ -119,14 +119,18 @@ public class MLPTrainer<P extends Serializable> extends MultiLabelDatasetTrainer
DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, double[]> lbExtractor) {
+ assert archSupplier != null;
+ assert loss!= null;
+ assert updatesStgy!= null;
+
try (Dataset<EmptyContext, SimpleLabeledDatasetData> dataset = datasetBuilder.build(
new EmptyContextBuilder<>(),
new SimpleLabeledDatasetDataBuilder<>(featureExtractor, lbExtractor)
)) {
MultilayerPerceptron mdl;
- if (lastLearnedModel != null) {
+ if (lastLearnedModel != null)
mdl = lastLearnedModel;
- } else {
+ else {
MLPArchitecture arch = archSupplier.apply(dataset);
mdl = new MultilayerPerceptron(arch, new RandomInitializer(seed));
}
@@ -205,6 +209,149 @@ public class MLPTrainer<P extends Serializable> extends MultiLabelDatasetTrainer
}
}
+ /**
+ * Get the multilayer perceptron architecture supplier that defines layers and activators.
+ *
+ * @return The property value.
+ */
+ public IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> getArchSupplier() {
+ return archSupplier;
+ }
+
+ /**
+ * Set up the multilayer perceptron architecture supplier that defines layers and activators.
+ *
+ * @param archSupplier The parameter value.
+ * @return Model with the multilayer perceptron architecture supplier that defines layers and activators.
+ */
+ public MLPTrainer<P> withArchSupplier(
+ IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier) {
+ this.archSupplier = archSupplier;
+ return this;
+ }
+
+ /**
+ * Get the loss function to be minimized during the training.
+ *
+ * @return The property value.
+ */
+ public IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> getLoss() {
+ return loss;
+ }
+
+ /**
+ * Set up the loss function to be minimized during the training.
+ *
+ * @param loss The parameter value.
+ * @return Model with the loss function to be minimized during the training.
+ */
+ public MLPTrainer<P> withLoss(
+ IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss) {
+ this.loss = loss;
+ return this;
+ }
+
+ /**
+ * Get the update strategy that defines how to update model parameters during the training.
+ *
+ * @return The property value.
+ */
+ public UpdatesStrategy<? super MultilayerPerceptron, P> getUpdatesStgy() {
+ return updatesStgy;
+ }
+
+ /**
+ * Set up the update strategy that defines how to update model parameters during the training.
+ *
+ * @param updatesStgy The parameter value.
+ * @return Model with the update strategy that defines how to update model parameters during the training.
+ */
+ public MLPTrainer<P> withUpdatesStgy(
+ UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy) {
+ this.updatesStgy = updatesStgy;
+ return this;
+ }
+
+ /**
+ * Get the maximal number of iterations before the training will be stopped.
+ *
+ * @return The property value.
+ */
+ public int getMaxIterations() {
+ return maxIterations;
+ }
+
+ /**
+ * Set up the maximal number of iterations before the training will be stopped.
+ *
+ * @param maxIterations The parameter value.
+ * @return Model with the maximal number of iterations before the training will be stopped.
+ */
+ public MLPTrainer<P> withMaxIterations(int maxIterations) {
+ this.maxIterations = maxIterations;
+ return this;
+ }
+
+ /**
+ * Get the batch size (per every partition).
+ *
+ * @return The property value.
+ */
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ /**
+ * Set up the batch size (per every partition).
+ *
+ * @param batchSize The parameter value.
+ * @return Model with the batch size (per every partition).
+ */
+ public MLPTrainer<P> withBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ return this;
+ }
+
+ /**
+ * Get the maximal number of local iterations before synchronization.
+ *
+ * @return The property value.
+ */
+ public int getLocIterations() {
+ return locIterations;
+ }
+
+ /**
+ * Set up the maximal number of local iterations before synchronization.
+ *
+ * @param locIterations The parameter value.
+ * @return Model with the maximal number of local iterations before synchronization.
+ */
+ public MLPTrainer<P> withLocIterations(int locIterations) {
+ this.locIterations = locIterations;
+ return this;
+ }
+
+ /**
+ * Get the multilayer perceptron model initializer.
+ *
+ * @return The property value.
+ */
+ public long getSeed() {
+ return seed;
+ }
+
+ /**
+ * Set up the multilayer perceptron model initializer.
+ *
+ * @param seed The parameter value.
+ * @return Model with the multilayer perceptron model initializer.
+ */
+ public MLPTrainer<P> withSeed(long seed) {
+ this.seed = seed;
+ return this;
+ }
+
/** {@inheritDoc} */
@Override protected boolean checkState(MultilayerPerceptron mdl) {
return true;
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationPreprocessor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationPreprocessor.java
index 8300820..2e1bd5c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationPreprocessor.java
@@ -68,8 +68,8 @@ public class BinarizationPreprocessor<K, V> implements IgniteBiFunction<K, V, Ve
return res;
}
- /** Gets the threshold parameter. */
- public double threshold() {
+ /** Get the threshold parameter. */
+ public double getThreshold() {
return threshold;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationTrainer.java
index 26541e0..ad8c90e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationTrainer.java
@@ -39,15 +39,16 @@ public class BinarizationTrainer<K, V> implements PreprocessingTrainer<K, V, Vec
}
/**
- * Gets the threshold parameter value.
- * @return The parameter value.
+ * Get the threshold parameter value.
+ *
+ * @return The property value.
*/
- public double threshold() {
+ public double getThreshold() {
return threshold;
}
/**
- * Sets the threshold parameter value.
+ * Set the threshold parameter value.
*
* @param threshold The given value.
* @return The Binarization trainer.
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java
index f716d96..a23d642 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java
@@ -222,4 +222,15 @@ public class EncoderTrainer<K, V> implements PreprocessingTrainer<K, V, Object[]
this.encoderType = type;
return this;
}
+
+ /**
+ * Sets the indices of features which should be encoded.
+ *
+ * @param handledIndices Indices of features which should be encoded.
+ * @return The changed trainer.
+ */
+ public EncoderTrainer<K, V> withEncoderType(Set<Integer> handledIndices) {
+ this.handledIndices = handledIndices;
+ return this;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
index 125ed24..4132d35 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
@@ -44,17 +44,17 @@ public class LinearRegressionSGDTrainer<P extends Serializable> extends SingleLa
/** Update strategy. */
private final UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy;
- /** Max number of iteration. */
+ /** Max amount of iterations. */
private int maxIterations = 1000;
/** Batch size. */
private int batchSize = 10;
- /** Number of local iterations. */
+ /** Amount of local iterations. */
private int locIterations = 100;
/** Seed for random generator. */
- private long seed = System.currentTimeMillis();
+ private long seed = 1234L;
/**
* Constructs a new instance of linear regression SGD trainer.
@@ -89,9 +89,12 @@ public class LinearRegressionSGDTrainer<P extends Serializable> extends SingleLa
}
/** {@inheritDoc} */
- @Override protected <K, V> LinearRegressionModel updateModel(LinearRegressionModel mdl, DatasetBuilder<K, V> datasetBuilder,
+ @Override protected <K, V> LinearRegressionModel updateModel(LinearRegressionModel mdl,
+ DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+ assert updatesStgy != null;
+
IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier = dataset -> {
int cols = dataset.compute(data -> {
@@ -162,7 +165,7 @@ public class LinearRegressionSGDTrainer<P extends Serializable> extends SingleLa
}
/**
- * Set up the max number of iterations before convergence.
+ * Set up the max amount of iterations before convergence.
*
* @param maxIterations The parameter value.
* @return Model with new max number of iterations before convergence parameter value.
@@ -204,4 +207,49 @@ public class LinearRegressionSGDTrainer<P extends Serializable> extends SingleLa
this.seed = seed;
return this;
}
+
+ /**
+ * Get the update strategy.
+ *
+ * @return The property value.
+ */
+ public UpdatesStrategy<? super MultilayerPerceptron, P> getUpdatesStgy() {
+ return updatesStgy;
+ }
+
+ /**
+ * Get the max amount of iterations.
+ *
+ * @return The property value.
+ */
+ public int getMaxIterations() {
+ return maxIterations;
+ }
+
+ /**
+ * Get the batch size.
+ *
+ * @return The property value.
+ */
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ /**
+ * Get the amount of local iterations.
+ *
+ * @return The property value.
+ */
+ public int getLocIterations() {
+ return locIterations;
+ }
+
+ /**
+ * Get the seed for random generator.
+ *
+ * @return The property value.
+ */
+ public long getSeed() {
+ return seed;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java
index 839dab5..fb5d5a0 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java
@@ -41,19 +41,19 @@ import org.jetbrains.annotations.NotNull;
*/
public class LogisticRegressionSGDTrainer<P extends Serializable> extends SingleLabelDatasetTrainer<LogisticRegressionModel> {
/** Update strategy. */
- private final UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy;
+ private UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy;
/** Max number of iteration. */
- private final int maxIterations;
+ private int maxIterations;
/** Batch size. */
- private final int batchSize;
+ private int batchSize;
/** Number of local iterations. */
- private final int locIterations;
+ private int locIterations;
/** Seed for random generator. */
- private final long seed;
+ private long seed;
/**
* Constructs a new instance of linear regression SGD trainer.
@@ -116,10 +116,11 @@ public class LogisticRegressionSGDTrainer<P extends Serializable> extends Single
IgniteBiFunction<K, V, double[]> lbExtractorWrapper = (k, v) -> new double[] {lbExtractor.apply(k, v)};
MultilayerPerceptron mlp;
- if(mdl != null) {
+ if (mdl != null) {
mlp = restoreMLPState(mdl);
mlp = trainer.update(mlp, datasetBuilder, featureExtractor, lbExtractorWrapper);
- } else
+ }
+ else
mlp = trainer.fit(datasetBuilder, featureExtractor, lbExtractorWrapper);
double[] params = mlp.parameters().getStorage().data();
@@ -136,8 +137,10 @@ public class LogisticRegressionSGDTrainer<P extends Serializable> extends Single
@NotNull private MultilayerPerceptron restoreMLPState(LogisticRegressionModel mdl) {
Vector weights = mdl.weights();
double intercept = mdl.intercept();
+
MLPArchitecture architecture1 = new MLPArchitecture(weights.size());
architecture1 = architecture1.withAddedLayer(1, true, Activators.SIGMOID);
+
MLPArchitecture architecture = architecture1;
MultilayerPerceptron perceptron = new MultilayerPerceptron(architecture);
@@ -145,6 +148,7 @@ public class LogisticRegressionSGDTrainer<P extends Serializable> extends Single
weights.nonZeroes().forEach(ith -> mlpState.set(ith.index(), ith.get()));
mlpState.set(mlpState.size() - 1, intercept);
perceptron.setParameters(mlpState);
+
return perceptron;
}
@@ -152,4 +156,93 @@ public class LogisticRegressionSGDTrainer<P extends Serializable> extends Single
@Override protected boolean checkState(LogisticRegressionModel mdl) {
return true;
}
+
+ /**
+ * Set up the max amount of iterations before convergence.
+ *
+ * @param maxIterations The parameter value.
+ * @return Model with new max number of iterations before convergence parameter value.
+ */
+ public LogisticRegressionSGDTrainer<P> withMaxIterations(int maxIterations) {
+ this.maxIterations = maxIterations;
+ return this;
+ }
+
+ /**
+ * Set up the batchSize parameter.
+ *
+ * @param batchSize The size of learning batch.
+ * @return Trainer with new batch size parameter value.
+ */
+ public LogisticRegressionSGDTrainer<P> withBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ return this;
+ }
+
+ /**
+ * Set up the amount of local iterations of SGD algorithm.
+ *
+ * @param amountOfLocIterations The parameter value.
+ * @return Trainer with new locIterations parameter value.
+ */
+ public LogisticRegressionSGDTrainer<P> withLocIterations(int amountOfLocIterations) {
+ this.locIterations = amountOfLocIterations;
+ return this;
+ }
+
+ /**
+ * Set up the random seed parameter.
+ *
+ * @param seed Seed for random generator.
+ * @return Trainer with new seed parameter value.
+ */
+ public LogisticRegressionSGDTrainer<P> withSeed(long seed) {
+ this.seed = seed;
+ return this;
+ }
+
+ /**
+ * Get the update strategy.
+ *
+ * @return The property value.
+ */
+ public UpdatesStrategy<? super MultilayerPerceptron, P> getUpdatesStgy() {
+ return updatesStgy;
+ }
+
+ /**
+ * Get the max amount of iterations.
+ *
+ * @return The property value.
+ */
+ public int getMaxIterations() {
+ return maxIterations;
+ }
+
+ /**
+ * Get the batch size.
+ *
+ * @return The property value.
+ */
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ /**
+ * Get the amount of local iterations.
+ *
+ * @return The property value.
+ */
+ public int getLocIterations() {
+ return locIterations;
+ }
+
+ /**
+ * Get the seed for random generator.
+ *
+ * @return The property value.
+ */
+ public long getSeed() {
+ return seed;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java
index eb44301..b9cdcc7 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java
@@ -77,14 +77,14 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
}
/** {@inheritDoc} */
- @Override public <K, V> LogRegressionMultiClassModel updateModel(LogRegressionMultiClassModel mdl,
+ @Override public <K, V> LogRegressionMultiClassModel updateModel(LogRegressionMultiClassModel newMdl,
DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
IgniteBiFunction<K, V, Double> lbExtractor) {
List<Double> classes = extractClassLabels(datasetBuilder, lbExtractor);
if(classes.isEmpty())
- return getLastTrainedModelOrThrowEmptyDatasetException(mdl);
+ return getLastTrainedModelOrThrowEmptyDatasetException(newMdl);
LogRegressionMultiClassModel multiClsMdl = new LogRegressionMultiClassModel();
@@ -101,12 +101,12 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
return 0.0;
};
- LogisticRegressionModel model = Optional.ofNullable(mdl)
+ LogisticRegressionModel mdl = Optional.ofNullable(newMdl)
.flatMap(multiClassModel -> multiClassModel.getModel(clsLb))
.map(learnedModel -> trainer.update(learnedModel, datasetBuilder, featureExtractor, lbTransformer))
.orElseGet(() -> trainer.fit(datasetBuilder, featureExtractor, lbTransformer));
- multiClsMdl.add(clsLb, model);
+ multiClsMdl.add(clsLb, mdl);
});
return multiClsMdl;
@@ -169,20 +169,20 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
}
/**
- * Gets the batch size.
+ * Get the batch size.
*
* @return The parameter value.
*/
- public double batchSize() {
+ public double getBatchSize() {
return batchSize;
}
/**
- * Gets the amount of outer iterations of SGD algorithm.
+ * Get the amount of outer iterations of SGD algorithm.
*
* @return The parameter value.
*/
- public int amountOfIterations() {
+ public int getAmountOfIterations() {
return amountOfIterations;
}
@@ -198,11 +198,11 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
}
/**
- * Gets the amount of local iterations.
+ * Get the amount of local iterations.
*
* @return The parameter value.
*/
- public int amountOfLocIterations() {
+ public int getAmountOfLocIterations() {
return amountOfLocIterations;
}
@@ -229,7 +229,7 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
}
/**
- * Gets the seed for random generator.
+ * Get the seed for random generator.
*
* @return The parameter value.
*/
@@ -249,11 +249,11 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
}
/**
- * Gets the update strategy..
+ * Get the update strategy..
*
* @return The parameter value.
*/
- public UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy() {
+ public UpdatesStrategy<? super MultilayerPerceptron, P> getUpdatesStgy() {
return updatesStgy;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/CrossValidation.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/CrossValidation.java b/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/CrossValidation.java
index 1ade876..ef4f30f 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/CrossValidation.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/selection/cv/CrossValidation.java
@@ -120,7 +120,7 @@ public class CrossValidation<M extends Model<Vector, L>, L, K, V> {
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, L> lbExtractor, int cv,
ParamGrid paramGrid) {
- List<Double[]> paramSets = new ParameterSetGenerator(paramGrid.getParamValuesByParamIndex()).generate();
+ List<Double[]> paramSets = new ParameterSetGenerator(paramGrid.getParamValuesByParamIdx()).generate();
CrossValidationResult cvRes = new CrossValidationResult();
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/selection/paramgrid/ParamGrid.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/selection/paramgrid/ParamGrid.java b/modules/ml/src/main/java/org/apache/ignite/ml/selection/paramgrid/ParamGrid.java
index 3279d93..f9c5bd2 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/selection/paramgrid/ParamGrid.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/selection/paramgrid/ParamGrid.java
@@ -25,17 +25,17 @@ import java.util.Map;
*/
public class ParamGrid {
/** Parameter values by parameter index. */
- private Map<Integer, Double[]> paramValuesByParamIndex = new HashMap<>();
+ private Map<Integer, Double[]> paramValuesByParamIdx = new HashMap<>();
/** Parameter names by parameter index. */
- private Map<Integer, String> paramNamesByParamIndex = new HashMap<>();
+ private Map<Integer, String> paramNamesByParamIdx = new HashMap<>();
/** Parameter counter. */
private int paramCntr;
/** */
- public Map<Integer, Double[]> getParamValuesByParamIndex() {
- return paramValuesByParamIndex;
+ public Map<Integer, Double[]> getParamValuesByParamIdx() {
+ return paramValuesByParamIdx;
}
/**
@@ -45,14 +45,14 @@ public class ParamGrid {
* @return The updated ParamGrid.
*/
public ParamGrid addHyperParam(String paramName, Double[] params) {
- paramValuesByParamIndex.put(paramCntr, params);
- paramNamesByParamIndex.put(paramCntr, paramName);
+ paramValuesByParamIdx.put(paramCntr, params);
+ paramNamesByParamIdx.put(paramCntr, paramName);
paramCntr++;
return this;
}
/** */
public String getParamNameByIndex(int idx) {
- return paramNamesByParamIndex.get(idx);
+ return paramNamesByParamIdx.get(idx);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
index 573df1a..8fb98d2 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
@@ -214,7 +214,7 @@ public class SVMLinearBinaryClassificationTrainer extends SingleLabelDatasetTrai
double qii = v.dot(v);
double newAlpha = calcNewAlpha(alpha, gradient, qii);
- Vector deltaWeights = v.times(lb * (newAlpha - alpha) / (this.lambda() * amountOfObservation));
+ Vector deltaWeights = v.times(lb * (newAlpha - alpha) / (this.getLambda() * amountOfObservation));
return new Deltas(newAlpha - alpha, deltaWeights);
}
@@ -233,7 +233,7 @@ public class SVMLinearBinaryClassificationTrainer extends SingleLabelDatasetTrai
/** */
private double calcGradient(double lb, Vector v, Vector weights, int amountOfObservation) {
double dotProduct = v.dot(weights);
- return (lb * dotProduct - 1.0) * (this.lambda() * amountOfObservation);
+ return (lb * dotProduct - 1.0) * (this.getLambda() * amountOfObservation);
}
/** */
@@ -261,18 +261,18 @@ public class SVMLinearBinaryClassificationTrainer extends SingleLabelDatasetTrai
}
/**
- * Gets the regularization lambda.
+ * Get the regularization lambda.
*
- * @return The parameter value.
+ * @return The property value.
*/
- public double lambda() {
+ public double getLambda() {
return lambda;
}
/**
- * Gets the amount of outer iterations of SCDA algorithm.
+ * Get the amount of outer iterations of SCDA algorithm.
*
- * @return The parameter value.
+ * @return The property value.
*/
public int getAmountOfIterations() {
return amountOfIterations;
@@ -290,9 +290,9 @@ public class SVMLinearBinaryClassificationTrainer extends SingleLabelDatasetTrai
}
/**
- * Gets the amount of local iterations of SCDA algorithm.
+ * Get the amount of local iterations of SCDA algorithm.
*
- * @return The parameter value.
+ * @return The property value.
*/
public int getAmountOfLocIterations() {
return amountOfLocIterations;
@@ -310,9 +310,9 @@ public class SVMLinearBinaryClassificationTrainer extends SingleLabelDatasetTrai
}
/**
- * Gets the seed number.
+ * Get the seed number.
*
- * @return The parameter value.
+ * @return The property value.
*/
public long getSeed() {
return seed;
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
index b77baa2..aeee178 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
@@ -82,9 +82,9 @@ public class SVMLinearMultiClassClassificationTrainer
classes.forEach(clsLb -> {
SVMLinearBinaryClassificationTrainer trainer = new SVMLinearBinaryClassificationTrainer()
- .withAmountOfIterations(this.amountOfIterations())
- .withAmountOfLocIterations(this.amountOfLocIterations())
- .withLambda(this.lambda())
+ .withAmountOfIterations(this.getAmountOfIterations())
+ .withAmountOfLocIterations(this.getAmountOfLocIterations())
+ .withLambda(this.getLambda())
.withSeed(this.seed);
IgniteBiFunction<K, V, Double> lbTransformer = (k, v) -> {
@@ -197,20 +197,20 @@ public class SVMLinearMultiClassClassificationTrainer
}
/**
- * Gets the regularization lambda.
+ * Get the regularization lambda.
*
- * @return The parameter value.
+ * @return The property value.
*/
- public double lambda() {
+ public double getLambda() {
return lambda;
}
/**
* Gets the amount of outer iterations of SCDA algorithm.
*
- * @return The parameter value.
+ * @return The property value.
*/
- public int amountOfIterations() {
+ public int getAmountOfIterations() {
return amountOfIterations;
}
@@ -228,9 +228,9 @@ public class SVMLinearMultiClassClassificationTrainer
/**
* Gets the amount of local iterations of SCDA algorithm.
*
- * @return The parameter value.
+ * @return The property value.
*/
- public int amountOfLocIterations() {
+ public int getAmountOfLocIterations() {
return amountOfLocIterations;
}
@@ -248,7 +248,7 @@ public class SVMLinearMultiClassClassificationTrainer
/**
* Gets the seed number.
*
- * @return The parameter value.
+ * @return The property value.
*/
public long getSeed() {
return seed;
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
index 355048a..45774cb 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
@@ -54,7 +54,7 @@ public abstract class DecisionTree<T extends ImpurityMeasure<T>> extends Dataset
private final DecisionTreeLeafBuilder decisionTreeLeafBuilder;
/** Use index structure instead of using sorting while learning. */
- protected boolean useIndex = true;
+ protected boolean usingIdx = true;
/**
* Constructs a new distributed decision tree trainer.
@@ -77,7 +77,7 @@ public abstract class DecisionTree<T extends ImpurityMeasure<T>> extends Dataset
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
try (Dataset<EmptyContext, DecisionTreeData> dataset = datasetBuilder.build(
new EmptyContextBuilder<>(),
- new DecisionTreeDataBuilder<>(featureExtractor, lbExtractor, useIndex)
+ new DecisionTreeDataBuilder<>(featureExtractor, lbExtractor, usingIdx)
)) {
return fit(dataset);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
index f8fc769..91ec8e1 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
@@ -91,7 +91,7 @@ public class DecisionTreeClassificationTrainer extends DecisionTree<GiniImpurity
* @return Decision tree trainer.
*/
public DecisionTreeClassificationTrainer withUseIndex(boolean useIndex) {
- this.useIndex = useIndex;
+ this.usingIdx = useIndex;
return this;
}
@@ -127,6 +127,6 @@ public class DecisionTreeClassificationTrainer extends DecisionTree<GiniImpurity
for (Double lb : labels)
encoder.put(lb, idx++);
- return new GiniImpurityMeasureCalculator(encoder, useIndex);
+ return new GiniImpurityMeasureCalculator(encoder, usingIdx);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
index 4c9aac9..ea57bcc 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
@@ -53,13 +53,13 @@ public class DecisionTreeRegressionTrainer extends DecisionTree<MSEImpurityMeasu
}
/**
- * Sets useIndex parameter and returns trainer instance.
+ * Sets usingIdx parameter and returns trainer instance.
*
- * @param useIndex Use index.
+ * @param usingIdx Use index.
* @return Decision tree trainer.
*/
- public DecisionTreeRegressionTrainer withUseIndex(boolean useIndex) {
- this.useIndex = useIndex;
+ public DecisionTreeRegressionTrainer withUsingIdx(boolean usingIdx) {
+ this.usingIdx = usingIdx;
return this;
}
@@ -67,6 +67,6 @@ public class DecisionTreeRegressionTrainer extends DecisionTree<MSEImpurityMeasu
@Override protected ImpurityMeasureCalculator<MSEImpurityMeasure> getImpurityMeasureCalculator(
Dataset<EmptyContext, DecisionTreeData> dataset) {
- return new MSEImpurityMeasureCalculator(useIndex);
+ return new MSEImpurityMeasureCalculator(usingIdx);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBBinaryClassifierOnTreesTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBBinaryClassifierOnTreesTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBBinaryClassifierOnTreesTrainer.java
index 4d87b47..b99dc2f 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBBinaryClassifierOnTreesTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBBinaryClassifierOnTreesTrainer.java
@@ -27,13 +27,13 @@ import org.jetbrains.annotations.NotNull;
*/
public class GDBBinaryClassifierOnTreesTrainer extends GDBBinaryClassifierTrainer {
/** Max depth. */
- private final int maxDepth;
+ private int maxDepth;
/** Min impurity decrease. */
- private final double minImpurityDecrease;
+ private double minImpurityDecrease;
- /** Use index structure instead of using sorting while learning. */
- private boolean useIndex = true;
+ /** Use index structure instead of using sorting during the learning process. */
+ private boolean usingIdx = true;
/**
* Constructs instance of GDBBinaryClassifierOnTreesTrainer.
@@ -53,22 +53,71 @@ public class GDBBinaryClassifierOnTreesTrainer extends GDBBinaryClassifierTraine
/** {@inheritDoc} */
@NotNull @Override protected DecisionTreeRegressionTrainer buildBaseModelTrainer() {
- return new DecisionTreeRegressionTrainer(maxDepth, minImpurityDecrease).withUseIndex(useIndex);
+ return new DecisionTreeRegressionTrainer(maxDepth, minImpurityDecrease).withUsingIdx(usingIdx);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected GDBLearningStrategy getLearningStrategy() {
+ return new GDBOnTreesLearningStrategy(usingIdx);
}
/**
- * Sets useIndex parameter and returns trainer instance.
+ * Set useIndex parameter and returns trainer instance.
*
- * @param useIndex Use index.
+ * @param usingIdx Use index.
* @return Decision tree trainer.
*/
- public GDBBinaryClassifierOnTreesTrainer withUseIndex(boolean useIndex) {
- this.useIndex = useIndex;
+ public GDBBinaryClassifierOnTreesTrainer withUsingIdx(boolean usingIdx) {
+ this.usingIdx = usingIdx;
return this;
}
- /** {@inheritDoc} */
- @Override protected GDBLearningStrategy getLearningStrategy() {
- return new GDBOnTreesLearningStrategy(useIndex);
+ /**
+ * Get the max depth.
+ *
+ * @return The property value.
+ */
+ public int getMaxDepth() {
+ return maxDepth;
+ }
+
+ /**
+ * Set up the max depth.
+ *
+ * @param maxDepth The parameter value.
+ * @return Decision tree trainer.
+ */
+ public GDBBinaryClassifierOnTreesTrainer setMaxDepth(int maxDepth) {
+ this.maxDepth = maxDepth;
+ return this;
+ }
+
+ /**
+ * Get the min impurity decrease.
+ *
+ * @return The property value.
+ */
+ public double getMinImpurityDecrease() {
+ return minImpurityDecrease;
+ }
+
+ /**
+ * Set up the min impurity decrease.
+ *
+ * @param minImpurityDecrease The parameter value.
+ * @return Decision tree trainer.
+ */
+ public GDBBinaryClassifierOnTreesTrainer setMinImpurityDecrease(double minImpurityDecrease) {
+ this.minImpurityDecrease = minImpurityDecrease;
+ return this;
+ }
+
+ /**
+ * Get the using index structure property instead of using sorting during the learning process.
+ *
+ * @return The property value.
+ */
+ public boolean isUsingIdx() {
+ return usingIdx;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBRegressionOnTreesTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBRegressionOnTreesTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBRegressionOnTreesTrainer.java
index e2a183c..b6c0b48 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBRegressionOnTreesTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/boosting/GDBRegressionOnTreesTrainer.java
@@ -27,13 +27,13 @@ import org.jetbrains.annotations.NotNull;
*/
public class GDBRegressionOnTreesTrainer extends GDBRegressionTrainer {
/** Max depth. */
- private final int maxDepth;
+ private int maxDepth;
/** Min impurity decrease. */
- private final double minImpurityDecrease;
+ private double minImpurityDecrease;
/** Use index structure instead of using sorting while learning. */
- private boolean useIndex = true;
+ private boolean usingIdx = true;
/**
* Constructs instance of GDBRegressionOnTreesTrainer.
@@ -53,22 +53,71 @@ public class GDBRegressionOnTreesTrainer extends GDBRegressionTrainer {
/** {@inheritDoc} */
@NotNull @Override protected DecisionTreeRegressionTrainer buildBaseModelTrainer() {
- return new DecisionTreeRegressionTrainer(maxDepth, minImpurityDecrease).withUseIndex(useIndex);
+ return new DecisionTreeRegressionTrainer(maxDepth, minImpurityDecrease).withUsingIdx(usingIdx);
}
/**
- * Sets useIndex parameter and returns trainer instance.
+ * Set useIndex parameter and returns trainer instance.
*
- * @param useIndex Use index.
+ * @param usingIdx Use index.
* @return Decision tree trainer.
*/
- public GDBRegressionOnTreesTrainer withUseIndex(boolean useIndex) {
- this.useIndex = useIndex;
+ public GDBRegressionOnTreesTrainer withUsingIdx(boolean usingIdx) {
+ this.usingIdx = usingIdx;
return this;
}
+ /**
+ * Get the max depth.
+ *
+ * @return The property value.
+ */
+ public int getMaxDepth() {
+ return maxDepth;
+ }
+
+ /**
+ * Set up the max depth.
+ *
+ * @param maxDepth The parameter value.
+ * @return Decision tree trainer.
+ */
+ public GDBRegressionOnTreesTrainer setMaxDepth(int maxDepth) {
+ this.maxDepth = maxDepth;
+ return this;
+ }
+
+ /**
+ * Get the min impurity decrease.
+ *
+ * @return The property value.
+ */
+ public double getMinImpurityDecrease() {
+ return minImpurityDecrease;
+ }
+
+ /**
+ * Set up the min impurity decrease.
+ *
+ * @param minImpurityDecrease The parameter value.
+ * @return Decision tree trainer.
+ */
+ public GDBRegressionOnTreesTrainer setMinImpurityDecrease(double minImpurityDecrease) {
+ this.minImpurityDecrease = minImpurityDecrease;
+ return this;
+ }
+
+ /**
+ * Get the using index structure property instead of using sorting during the learning process.
+ *
+ * @return The property value.
+ */
+ public boolean isUsingIdx() {
+ return usingIdx;
+ }
+
/** {@inheritDoc} */
@Override protected GDBLearningStrategy getLearningStrategy() {
- return new GDBOnTreesLearningStrategy(useIndex);
+ return new GDBOnTreesLearningStrategy(usingIdx);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java
index 91fcf0a..c617d8d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java
@@ -76,7 +76,7 @@ public abstract class RandomForestTrainer<L, S extends ImpurityComputer<Bootstra
private int cntOfTrees = 1;
/** Subsample size. */
- private double subsampleSize = 1.0;
+ private double subSampleSize = 1.0;
/** Max depth. */
private int maxDepth = 5;
@@ -88,10 +88,10 @@ public abstract class RandomForestTrainer<L, S extends ImpurityComputer<Bootstra
private List<FeatureMeta> meta;
/** Features per tree. */
- private int featuresPerTree;
+ private int featuresPerTree = 5;
/** Seed. */
- private long seed = System.currentTimeMillis();
+ private long seed = 1234L;
/** Random generator. */
private Random random = new Random(seed);
@@ -115,7 +115,7 @@ public abstract class RandomForestTrainer<L, S extends ImpurityComputer<Bootstra
List<TreeRoot> models = null;
try (Dataset<EmptyContext, BootstrappedDatasetPartition> dataset = datasetBuilder.build(
new EmptyContextBuilder<>(),
- new BootstrappedDatasetBuilder<>(featureExtractor, lbExtractor, cntOfTrees, subsampleSize))) {
+ new BootstrappedDatasetBuilder<>(featureExtractor, lbExtractor, cntOfTrees, subSampleSize))) {
if(!init(dataset))
return buildComposition(Collections.emptyList());
@@ -144,11 +144,11 @@ public abstract class RandomForestTrainer<L, S extends ImpurityComputer<Bootstra
}
/**
- * @param subsampleSize Subsample size.
+ * @param subSampleSize Subsample size.
* @return an instance of current object with valid type in according to inheritance.
*/
- public T withSubsampleSize(double subsampleSize) {
- this.subsampleSize = subsampleSize;
+ public T withSubSampleSize(double subSampleSize) {
+ this.subSampleSize = subSampleSize;
return instance();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansModelTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansModelTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansModelTest.java
index 03e0e6d..f71b7b3 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansModelTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansModelTest.java
@@ -54,7 +54,7 @@ public class KMeansModelTest {
Assert.assertEquals(mdl.apply(new DenseVector(new double[]{-1.1, -1.1})), 3.0, PRECISION);
Assert.assertEquals(mdl.distanceMeasure(), distanceMeasure);
- Assert.assertEquals(mdl.amountOfClusters(), 4);
- Assert.assertArrayEquals(mdl.centers(), centers);
+ Assert.assertEquals(mdl.getAmountOfClusters(), 4);
+ Assert.assertArrayEquals(mdl.getCenters(), centers);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java
index 03f044a..74ff8f1 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java
@@ -58,7 +58,7 @@ public class KMeansTrainerTest {
@Test
public void findOneClusters() {
KMeansTrainer trainer = createAndCheckTrainer();
- KMeansModel knnMdl = trainer.withK(1).fit(
+ KMeansModel knnMdl = trainer.withAmountOfClusters(1).fit(
new LocalDatasetBuilder<>(data, 2),
(k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
(k, v) -> v[2]
@@ -76,7 +76,7 @@ public class KMeansTrainerTest {
@Test
public void testUpdateMdl() {
KMeansTrainer trainer = createAndCheckTrainer();
- KMeansModel originalMdl = trainer.withK(1).fit(
+ KMeansModel originalMdl = trainer.withAmountOfClusters(1).fit(
new LocalDatasetBuilder<>(data, 2),
(k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
(k, v) -> v[2]
@@ -106,11 +106,11 @@ public class KMeansTrainerTest {
@NotNull private KMeansTrainer createAndCheckTrainer() {
KMeansTrainer trainer = new KMeansTrainer()
.withDistance(new EuclideanDistance())
- .withK(10)
+ .withAmountOfClusters(10)
.withMaxIterations(1)
.withEpsilon(PRECISION)
.withSeed(2);
- assertEquals(10, trainer.getK());
+ assertEquals(10, trainer.getAmountOfClusters());
assertEquals(2, trainer.getSeed());
assertTrue(trainer.getDistance() instanceof EuclideanDistance);
return trainer;
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/test/java/org/apache/ignite/ml/common/LocalModelsTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/common/LocalModelsTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/common/LocalModelsTest.java
index 9315850..ca3f0b5 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/common/LocalModelsTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/common/LocalModelsTest.java
@@ -196,7 +196,7 @@ public class LocalModelsTest {
data.put(1, new double[] {1.0, 1960, 373200});
KMeansTrainer trainer = new KMeansTrainer()
- .withK(1);
+ .withAmountOfClusters(1);
return trainer.fit(
new LocalDatasetBuilder<>(data, 2),
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java
index 89b8c9c..4c3655b 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java
@@ -57,7 +57,7 @@ public class GDBTrainerTest {
}
GDBTrainer trainer = new GDBRegressionOnTreesTrainer(1.0, 2000, 3, 0.0)
- .withUseIndex(true);
+ .withUsingIdx(true);
Model<Vector, Double> mdl = trainer.fit(
learningSample, 1,
@@ -131,7 +131,7 @@ public class GDBTrainerTest {
learningSample.put(i, new double[] {xs[i], ys[i]});
GDBTrainer trainer = new GDBBinaryClassifierOnTreesTrainer(0.3, 500, 3, 0.0)
- .withUseIndex(true)
+ .withUsingIdx(true)
.withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.3));
Model<Vector, Double> mdl = fitter.apply(trainer, learningSample);
@@ -177,10 +177,10 @@ public class GDBTrainerTest {
IgniteBiFunction<Integer, double[], Double> lExtr = (k, v) -> v[1];
GDBTrainer classifTrainer = new GDBBinaryClassifierOnTreesTrainer(0.3, 500, 3, 0.0)
- .withUseIndex(true)
+ .withUsingIdx(true)
.withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.3));
GDBTrainer regressTrainer = new GDBRegressionOnTreesTrainer(0.3, 500, 3, 0.0)
- .withUseIndex(true)
+ .withUsingIdx(true)
.withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.3));
testUpdate(learningSample, fExtr, lExtr, classifTrainer);
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java
index b06fd67..7e5a079 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/environment/LearningEnvironmentTest.java
@@ -45,7 +45,7 @@ public class LearningEnvironmentTest {
.withFeaturesCountSelectionStrgy(FeaturesCountSelectionStrategies.ONE_THIRD)
.withMaxDepth(4)
.withMinImpurityDelta(0.)
- .withSubsampleSize(0.3)
+ .withSubSampleSize(0.3)
.withSeed(0);
LearningEnvironment environment = LearningEnvironment.builder()
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationTrainerTest.java
index 857d9bd..d465e82 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/binarization/BinarizationTrainerTest.java
@@ -69,14 +69,14 @@ public class BinarizationTrainerTest {
BinarizationTrainer<Integer, double[]> binarizationTrainer = new BinarizationTrainer<Integer, double[]>()
.withThreshold(10);
- assertEquals(10., binarizationTrainer.threshold(), 0);
+ assertEquals(10., binarizationTrainer.getThreshold(), 0);
BinarizationPreprocessor<Integer, double[]> preprocessor = binarizationTrainer.fit(
datasetBuilder,
(k, v) -> VectorUtils.of(v)
);
- assertEquals(binarizationTrainer.threshold(), preprocessor.threshold(), 0);
+ assertEquals(binarizationTrainer.getThreshold(), preprocessor.getThreshold(), 0);
assertArrayEquals(new double[] {0, 0, 1}, preprocessor.apply(5, new double[] {1, 10, 100}).asArray(), 1e-8);
}
@@ -93,7 +93,7 @@ public class BinarizationTrainerTest {
BinarizationTrainer<Integer, double[]> binarizationTrainer = new BinarizationTrainer<Integer, double[]>()
.withThreshold(10);
- assertEquals(10., binarizationTrainer.threshold(), 0);
+ assertEquals(10., binarizationTrainer.getThreshold(), 0);
IgniteBiFunction<Integer, double[], Vector> preprocessor = binarizationTrainer.fit(
data,
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java
index f08501c..73c8842 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java
@@ -61,11 +61,11 @@ public class LogRegMultiClassTrainerTest extends TrainerTest {
.withBatchSize(100)
.withSeed(123L);
- Assert.assertEquals(trainer.amountOfIterations(), 1000);
- Assert.assertEquals(trainer.amountOfLocIterations(), 10);
- Assert.assertEquals(trainer.batchSize(), 100, PRECISION);
+ Assert.assertEquals(trainer.getAmountOfIterations(), 1000);
+ Assert.assertEquals(trainer.getAmountOfLocIterations(), 10);
+ Assert.assertEquals(trainer.getBatchSize(), 100, PRECISION);
Assert.assertEquals(trainer.seed(), 123L);
- Assert.assertEquals(trainer.updatesStgy(), stgy);
+ Assert.assertEquals(trainer.getUpdatesStgy(), stgy);
LogRegressionMultiClassModel mdl = trainer.fit(
cacheMock,
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
index 4e64925..84975a8 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
@@ -73,7 +73,7 @@ public class DecisionTreeRegressionTrainerTest {
}
DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(1, 0)
- .withUseIndex(useIndex == 1);
+ .withUsingIdx(useIndex == 1);
DecisionTreeNode tree = trainer.fit(
data,
http://git-wip-us.apache.org/repos/asf/ignite/blob/b10ba044/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestTest.java
index ed474fe..9fa7f0e 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestTest.java
@@ -60,7 +60,7 @@ public class RandomForestTest {
.withFeaturesCountSelectionStrgy(x -> 4)
.withMaxDepth(maxDepth)
.withMinImpurityDelta(minImpDelta)
- .withSubsampleSize(0.1);
+ .withSubSampleSize(0.1);
/** */
@Test
[16/50] [abbrv] ignite git commit: IGNITE-9382: Fix Node.js client o
process large payloads. - Fixes #4629.
Posted by ag...@apache.org.
IGNITE-9382: Fix Node.js client o process large payloads. - Fixes #4629.
Signed-off-by: shroman <rs...@yahoo.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/08c563a1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/08c563a1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/08c563a1
Branch: refs/heads/ignite-5960
Commit: 08c563a14f90702bf54b22d0b74a49e03f3df760
Parents: e8aa038
Author: Toru Yabuki <ya...@gmail.com>
Authored: Tue Sep 4 14:52:03 2018 +0900
Committer: shroman <rs...@yahoo.com>
Committed: Tue Sep 4 14:52:03 2018 +0900
----------------------------------------------------------------------
.../nodejs/lib/internal/ClientSocket.js | 35 +++++++++++++++-----
.../nodejs/lib/internal/MessageBuffer.js | 6 ++++
2 files changed, 32 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/08c563a1/modules/platforms/nodejs/lib/internal/ClientSocket.js
----------------------------------------------------------------------
diff --git a/modules/platforms/nodejs/lib/internal/ClientSocket.js b/modules/platforms/nodejs/lib/internal/ClientSocket.js
index 1d1a4dd..73e11f0 100644
--- a/modules/platforms/nodejs/lib/internal/ClientSocket.js
+++ b/modules/platforms/nodejs/lib/internal/ClientSocket.js
@@ -110,6 +110,8 @@ class ClientSocket {
this._onSocketDisconnect = onSocketDisconnect;
this._error = null;
this._wasConnected = false;
+ this._buffer = null;
+ this._offset = 0;
}
async connect() {
@@ -198,28 +200,43 @@ class ClientSocket {
if (this._state === STATE.DISCONNECTED) {
return;
}
- let offset = 0;
- while (offset < message.length) {
- let buffer = MessageBuffer.from(message, offset);
+ if (this._buffer) {
+ this._buffer.concat(message);
+ this._buffer.position = this._offset;
+ }
+ else {
+ this._buffer = MessageBuffer.from(message, 0);
+ }
+ while (this._buffer && this._offset < this._buffer.length) {
// Response length
- const length = buffer.readInteger();
- offset += length + BinaryUtils.getSize(BinaryUtils.TYPE_CODE.INTEGER);
+ const length = this._buffer.readInteger() + BinaryUtils.getSize(BinaryUtils.TYPE_CODE.INTEGER);
+ if (this._buffer.length < this._offset + length) {
+ break;
+ }
+ this._offset += length;
+
let requestId, isSuccess;
const isHandshake = this._state === STATE.HANDSHAKE;
if (isHandshake) {
// Handshake status
- isSuccess = (buffer.readByte() === HANDSHAKE_SUCCESS_STATUS_CODE)
+ isSuccess = (this._buffer.readByte() === HANDSHAKE_SUCCESS_STATUS_CODE);
requestId = this._handshakeRequestId.toString();
}
else {
// Request id
- requestId = buffer.readLong().toString();
+ requestId = this._buffer.readLong().toString();
// Status code
- isSuccess = (buffer.readInteger() === REQUEST_SUCCESS_STATUS_CODE);
+ isSuccess = (this._buffer.readInteger() === REQUEST_SUCCESS_STATUS_CODE);
}
- this._logMessage(requestId, false, buffer.data);
+ this._logMessage(requestId, false, this._buffer.data);
+
+ const buffer = this._buffer;
+ if (this._offset === this._buffer.length) {
+ this._buffer = null;
+ this._offset = 0;
+ }
if (this._requests.has(requestId)) {
const request = this._requests.get(requestId);
http://git-wip-us.apache.org/repos/asf/ignite/blob/08c563a1/modules/platforms/nodejs/lib/internal/MessageBuffer.js
----------------------------------------------------------------------
diff --git a/modules/platforms/nodejs/lib/internal/MessageBuffer.js b/modules/platforms/nodejs/lib/internal/MessageBuffer.js
index f1407bf..dc3e928 100644
--- a/modules/platforms/nodejs/lib/internal/MessageBuffer.js
+++ b/modules/platforms/nodejs/lib/internal/MessageBuffer.js
@@ -42,6 +42,12 @@ class MessageBuffer {
return buf;
}
+ concat(source) {
+ this._buffer = Buffer.concat([this._buffer, source]);
+ this._length = this._buffer.length;
+ this._capacity = this._length;
+ }
+
get position() {
return this._position;
}
[03/50] [abbrv] ignite git commit: IGNITE-8568 Web Console: Added
support for "Collocated" query mode on "Queries" screen.
Posted by ag...@apache.org.
IGNITE-8568 Web Console: Added support for "Collocated" query mode on "Queries" screen.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d1b38824
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d1b38824
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d1b38824
Branch: refs/heads/ignite-5960
Commit: d1b388245aabd54bff227bf5a6e3d9ed8f993915
Parents: 271c7cd
Author: Vasiliy Sisko <vs...@gridgain.com>
Authored: Fri Aug 31 17:49:55 2018 +0700
Committer: Andrey Novikov <an...@gridgain.com>
Committed: Fri Aug 31 17:49:55 2018 +0700
----------------------------------------------------------------------
.../page-queries/components/queries-notebook/controller.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d1b38824/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
index 5ba7a27..d6a8271 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
@@ -41,7 +41,7 @@ const ROW_IDX = {value: -2, type: 'java.lang.Integer', label: 'ROW_IDX'};
const NON_COLLOCATED_JOINS_SINCE = '1.7.0';
-const COLLOCATED_QUERY_SINCE = [['2.3.5', '2.4.0'], ['2.4.6', '2.5.0'], '2.5.2'];
+const COLLOCATED_QUERY_SINCE = [['2.3.5', '2.4.0'], ['2.4.6', '2.5.0'], '2.5.1-p13'];
const ENFORCE_JOIN_SINCE = [['1.7.9', '1.8.0'], ['1.8.4', '1.9.0'], '1.9.1'];
[11/50] [abbrv] ignite git commit: IGNITE-9348: ML examples
improvements
Posted by ag...@apache.org.
IGNITE-9348: ML examples improvements
this closes #4662
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4c273204
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4c273204
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4c273204
Branch: refs/heads/ignite-5960
Commit: 4c273204b712b62e9aa5be5d83e08045b3863dd6
Parents: d73e035
Author: Oleg Ignatenko <oi...@gridgain.com>
Authored: Mon Sep 3 13:04:50 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Mon Sep 3 13:04:50 2018 +0300
----------------------------------------------------------------------
examples/DEVNOTES.txt | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4c273204/examples/DEVNOTES.txt
----------------------------------------------------------------------
diff --git a/examples/DEVNOTES.txt b/examples/DEVNOTES.txt
new file mode 100644
index 0000000..ba578bb
--- /dev/null
+++ b/examples/DEVNOTES.txt
@@ -0,0 +1,11 @@
+Ignite Examples Build Instructions
+==================================
+1) Compile and install Ignite from project root folder:
+
+ mvn clean install -Pall-java,all-scala,licenses -DskipTests
+
+ (If needed, refer DEVNOTES.txt in project root folder for most up-to-date build instructions.)
+
+2) Build examples from "examples" sub-folder under Ignite project root:
+
+ mvn clean package -DskipTests
[43/50] [abbrv] ignite git commit: IGNITE-9483: MVCC compatibility
fixes. This closes #4696.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaPrimaryKeysRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaPrimaryKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaPrimaryKeysRequest.java
index 957225a..def53b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaPrimaryKeysRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaPrimaryKeysRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.Nullable;
@@ -66,16 +67,18 @@ public class JdbcMetaPrimaryKeysRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeString(schemaName);
writer.writeString(tblName);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
schemaName = reader.readString();
tblName = reader.readString();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaPrimaryKeysResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaPrimaryKeysResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaPrimaryKeysResult.java
index bd0dd90..f6d986e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaPrimaryKeysResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaPrimaryKeysResult.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -51,8 +52,9 @@ public class JdbcMetaPrimaryKeysResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
if (F.isEmpty(meta))
writer.writeInt(0);
@@ -60,13 +62,14 @@ public class JdbcMetaPrimaryKeysResult extends JdbcResult {
writer.writeInt(meta.size());
for(JdbcPrimaryKeyMeta m : meta)
- m.writeBinary(writer);
+ m.writeBinary(writer, ver);
}
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
int size = reader.readInt();
@@ -78,7 +81,7 @@ public class JdbcMetaPrimaryKeysResult extends JdbcResult {
for (int i = 0; i < size; ++i) {
JdbcPrimaryKeyMeta m = new JdbcPrimaryKeyMeta();
- m.readBinary(reader);
+ m.readBinary(reader, ver);
meta.add(m);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaSchemasRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaSchemasRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaSchemasRequest.java
index 43bbe5d..715c798 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaSchemasRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaSchemasRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -53,15 +54,17 @@ public class JdbcMetaSchemasRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeString(schemaName);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
this.schemaName = reader.readString();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaSchemasResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaSchemasResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaSchemasResult.java
index 48b6aae..8d24a41 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaSchemasResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaSchemasResult.java
@@ -21,6 +21,7 @@ import java.util.Collection;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -46,15 +47,17 @@ public class JdbcMetaSchemasResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
JdbcUtils.writeStringCollection(writer, schemas);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
schemas = JdbcUtils.readStringList(reader);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaTablesRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaTablesRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaTablesRequest.java
index 740b656..3ea6c35 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaTablesRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaTablesRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -65,16 +66,18 @@ public class JdbcMetaTablesRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeString(schemaName);
writer.writeString(tblName);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
this.schemaName = reader.readString();
this.tblName = reader.readString();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaTablesResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaTablesResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaTablesResult.java
index 585667e..d5afb7c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaTablesResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaTablesResult.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -49,8 +50,9 @@ public class JdbcMetaTablesResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
if (F.isEmpty(meta))
writer.writeInt(0);
@@ -58,13 +60,14 @@ public class JdbcMetaTablesResult extends JdbcResult {
writer.writeInt(meta.size());
for(JdbcTableMeta m : meta)
- m.writeBinary(writer);
+ m.writeBinary(writer, ver);
}
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
int size = reader.readInt();
@@ -76,7 +79,7 @@ public class JdbcMetaTablesResult extends JdbcResult {
for (int i = 0; i < size; ++i) {
JdbcTableMeta m = new JdbcTableMeta();
- m.readBinary(reader);
+ m.readBinary(reader, ver);
meta.add(m);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcOrderedBatchExecuteRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcOrderedBatchExecuteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcOrderedBatchExecuteRequest.java
index 1d4d4ac..743978d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcOrderedBatchExecuteRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcOrderedBatchExecuteRequest.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.NotNull;
@@ -61,15 +62,15 @@ public class JdbcOrderedBatchExecuteRequest extends JdbcBatchExecuteRequest
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer, ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeLong(order);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader, ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
order = reader.readLong();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcOrderedBatchExecuteResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcOrderedBatchExecuteResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcOrderedBatchExecuteResult.java
index 84853d4..76f665f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcOrderedBatchExecuteResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcOrderedBatchExecuteResult.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -54,16 +55,18 @@ public class JdbcOrderedBatchExecuteResult extends JdbcBatchExecuteResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeLong(order);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
order = reader.readLong();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcParameterMeta.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcParameterMeta.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcParameterMeta.java
index c0cfc9e..52e5a19 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcParameterMeta.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcParameterMeta.java
@@ -22,6 +22,7 @@ import java.sql.SQLException;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -133,7 +134,8 @@ public class JdbcParameterMeta implements JdbcRawBinarylizable {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
writer.writeInt(isNullable);
writer.writeBoolean(signed);
writer.writeInt(precision);
@@ -145,7 +147,8 @@ public class JdbcParameterMeta implements JdbcRawBinarylizable {
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
isNullable = reader.readInt();
signed = reader.readBoolean();
precision = reader.readInt();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcPrimaryKeyMeta.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcPrimaryKeyMeta.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcPrimaryKeyMeta.java
index 6b9bf70..dffbdca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcPrimaryKeyMeta.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcPrimaryKeyMeta.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.F;
/**
@@ -89,7 +90,8 @@ public class JdbcPrimaryKeyMeta implements JdbcRawBinarylizable {
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
writer.writeString(schemaName);
writer.writeString(tblName);
writer.writeString(name);
@@ -98,7 +100,8 @@ public class JdbcPrimaryKeyMeta implements JdbcRawBinarylizable {
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
schemaName = reader.readString();
tblName = reader.readString();
name = reader.readString();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQuery.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQuery.java
index f7ffb99..d9960da 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQuery.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQuery.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.processors.odbc.SqlListenerUtils;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -63,7 +64,8 @@ public class JdbcQuery implements JdbcRawBinarylizable {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) {
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) {
writer.writeString(sql);
if (args == null || args.length == 0)
@@ -77,7 +79,8 @@ public class JdbcQuery implements JdbcRawBinarylizable {
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) {
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) {
sql = reader.readString();
int argsNum = reader.readInt();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryCloseRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryCloseRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryCloseRequest.java
index 872889c..5c631c3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryCloseRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryCloseRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -52,15 +53,17 @@ public class JdbcQueryCloseRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeLong(queryId);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
queryId = reader.readLong();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteMultipleStatementsResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteMultipleStatementsResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteMultipleStatementsResult.java
index 9bbdd59..44a56aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteMultipleStatementsResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteMultipleStatementsResult.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -80,14 +81,15 @@ public class JdbcQueryExecuteMultipleStatementsResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
if (results != null && results.size() > 0) {
writer.writeInt(results.size());
for (JdbcResultInfo r : results)
- r.writeBinary(writer);
+ r.writeBinary(writer, ver);
if (results.get(0).isQuery()) {
writer.writeBoolean(last);
@@ -101,8 +103,9 @@ public class JdbcQueryExecuteMultipleStatementsResult extends JdbcResult {
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
int cnt = reader.readInt();
@@ -114,7 +117,7 @@ public class JdbcQueryExecuteMultipleStatementsResult extends JdbcResult {
for (int i = 0; i < cnt; ++i) {
JdbcResultInfo r = new JdbcResultInfo();
- r.readBinary(reader);
+ r.readBinary(reader, ver);
results.add(r);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteRequest.java
index c13117a..e4f3398 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteRequest.java
@@ -21,12 +21,15 @@ import java.io.IOException;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.processors.odbc.SqlListenerUtils;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.internal.processors.odbc.jdbc.JdbcConnectionContext.VER_2_7_0;
+
/**
* JDBC query execute request.
*/
@@ -58,6 +61,8 @@ public class JdbcQueryExecuteRequest extends JdbcRequest {
*/
JdbcQueryExecuteRequest() {
super(QRY_EXEC);
+
+ autoCommit = true;
}
/**
@@ -132,8 +137,9 @@ public class JdbcQueryExecuteRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeString(schemaName);
writer.writeInt(pageSize);
@@ -147,15 +153,17 @@ public class JdbcQueryExecuteRequest extends JdbcRequest {
SqlListenerUtils.writeObject(writer, arg, false);
}
- writer.writeByte((byte)stmtType.ordinal());
+ if (ver.compareTo(VER_2_7_0) >= 0)
+ writer.writeBoolean(autoCommit);
- writer.writeBoolean(autoCommit);
+ writer.writeByte((byte)stmtType.ordinal());
}
/** {@inheritDoc} */
@SuppressWarnings("SimplifiableIfStatement")
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
schemaName = reader.readString();
pageSize = reader.readInt();
@@ -169,6 +177,9 @@ public class JdbcQueryExecuteRequest extends JdbcRequest {
for (int i = 0; i < argsNum; ++i)
args[i] = SqlListenerUtils.readObject(reader, false);
+ if (ver.compareTo(VER_2_7_0) >= 0)
+ autoCommit = reader.readBoolean();
+
try {
if (reader.available() > 0)
stmtType = JdbcStatementType.fromOrdinal(reader.readByte());
@@ -178,16 +189,6 @@ public class JdbcQueryExecuteRequest extends JdbcRequest {
catch (IOException e) {
throw new BinaryObjectException(e);
}
-
- try {
- if (reader.available() > 0)
- autoCommit = reader.readBoolean();
- else
- autoCommit = true;
- }
- catch (IOException e) {
- throw new BinaryObjectException(e);
- }
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteResult.java
index fdebdb8..342e8ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryExecuteResult.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -112,8 +113,9 @@ public class JdbcQueryExecuteResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeLong(queryId);
writer.writeBoolean(isQuery);
@@ -131,8 +133,9 @@ public class JdbcQueryExecuteResult extends JdbcResult {
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
queryId = reader.readLong();
isQuery = reader.readBoolean();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryFetchRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryFetchRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryFetchRequest.java
index 776c3bf..59ed9a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryFetchRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryFetchRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -65,16 +66,18 @@ public class JdbcQueryFetchRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeLong(queryId);
writer.writeInt(pageSize);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
queryId = reader.readLong();
pageSize = reader.readInt();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryFetchResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryFetchResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryFetchResult.java
index ac4a603..e62efcb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryFetchResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryFetchResult.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -66,8 +67,9 @@ public class JdbcQueryFetchResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeBoolean(last);
@@ -75,8 +77,9 @@ public class JdbcQueryFetchResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
last = reader.readBoolean();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryMetadataRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryMetadataRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryMetadataRequest.java
index bdef321..f30ecfd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryMetadataRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryMetadataRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -53,15 +54,17 @@ public class JdbcQueryMetadataRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeLong(qryId);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
qryId = reader.readLong();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryMetadataResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryMetadataResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryMetadataResult.java
index c8c0991..d366e60 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryMetadataResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcQueryMetadataResult.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -58,8 +59,9 @@ public class JdbcQueryMetadataResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
if (F.isEmpty(meta))
writer.writeInt(0);
@@ -67,13 +69,14 @@ public class JdbcQueryMetadataResult extends JdbcResult {
writer.writeInt(meta.size());
for (JdbcColumnMeta m : meta)
- m.writeBinary(writer);
+ m.writeBinary(writer, ver);
}
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
int size = reader.readInt();
@@ -85,7 +88,7 @@ public class JdbcQueryMetadataResult extends JdbcResult {
for (int i = 0; i < size; ++i) {
JdbcColumnMeta m = new JdbcColumnMeta();
- m.readBinary(reader);
+ m.readBinary(reader, ver);
meta.add(m);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRawBinarylizable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRawBinarylizable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRawBinarylizable.java
index c3f1874..22c514d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRawBinarylizable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRawBinarylizable.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
/**
* Interface that allows to implement custom serialization
@@ -30,15 +31,18 @@ public interface JdbcRawBinarylizable {
* Writes fields to provided writer.
*
* @param writer Binary object writer.
+ * @param ver ver Protocol version.
* @throws BinaryObjectException In case of error.
*/
- public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException;
+ public void writeBinary(BinaryWriterExImpl writer, ClientListenerProtocolVersion ver) throws BinaryObjectException;
/**
* Reads fields from provided reader.
*
* @param reader Binary object reader.
+ * @param ver Protocol version.
* @throws BinaryObjectException In case of error.
*/
- public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException;
+ public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequest.java
index 3d5b869..0674edf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.processors.odbc.ClientListenerRequestNoId;
/**
@@ -77,12 +78,14 @@ public class JdbcRequest extends ClientListenerRequestNoId implements JdbcRawBin
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
writer.writeByte(type);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
// No-op.
}
@@ -95,10 +98,12 @@ public class JdbcRequest extends ClientListenerRequestNoId implements JdbcRawBin
/**
* @param reader Binary reader.
+ * @param ver Protocol version.
* @return Request object.
* @throws BinaryObjectException On error.
*/
- public static JdbcRequest readRequest(BinaryReaderExImpl reader) throws BinaryObjectException {
+ public static JdbcRequest readRequest(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
int reqType = reader.readByte();
JdbcRequest req;
@@ -173,7 +178,7 @@ public class JdbcRequest extends ClientListenerRequestNoId implements JdbcRawBin
throw new IgniteException("Unknown SQL listener request ID: [request ID=" + reqType + ']');
}
- req.readBinary(reader);
+ req.readBinary(reader, ver);
return req;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResponse.java
index 91f26d8..5d5b4e3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResponse.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -76,14 +77,15 @@ public class JdbcResponse extends ClientListenerResponse implements JdbcRawBinar
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
writer.writeInt(status());
if (status() == STATUS_SUCCESS) {
writer.writeBoolean(res != null);
if (res != null)
- res.writeBinary(writer);
+ res.writeBinary(writer, ver);
}
else
writer.writeString(error());
@@ -91,12 +93,13 @@ public class JdbcResponse extends ClientListenerResponse implements JdbcRawBinar
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
status(reader.readInt());
if (status() == STATUS_SUCCESS) {
if (reader.readBoolean())
- res = JdbcResult.readResult(reader);
+ res = JdbcResult.readResult(reader, ver);
}
else
error(reader.readString());
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResult.java
index 199e5da..3a0d7bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResult.java
@@ -21,6 +21,7 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
/**
* JDBC response result.
@@ -87,21 +88,24 @@ public class JdbcResult implements JdbcRawBinarylizable {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
writer.writeByte(type);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
// No-op.
}
/**
* @param reader Binary reader.
+ * @param ver Protocol verssion.
* @return Request object.
* @throws BinaryObjectException On error.
*/
- public static JdbcResult readResult(BinaryReaderExImpl reader) throws BinaryObjectException {
+ public static JdbcResult readResult(BinaryReaderExImpl reader, ClientListenerProtocolVersion ver) throws BinaryObjectException {
int resId = reader.readByte();
JdbcResult res;
@@ -191,7 +195,7 @@ public class JdbcResult implements JdbcRawBinarylizable {
throw new IgniteException("Unknown SQL listener request ID: [request ID=" + resId + ']');
}
- res.readBinary(reader);
+ res.readBinary(reader, ver);
return res;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResultInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResultInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResultInfo.java
index f0706e4..5fab77a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResultInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcResultInfo.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -75,14 +76,16 @@ public class JdbcResultInfo implements JdbcRawBinarylizable {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) {
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) {
writer.writeBoolean(isQuery);
writer.writeLong(updCnt);
writer.writeLong(qryId);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) {
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) {
isQuery = reader.readBoolean();
updCnt = reader.readLong();
qryId = reader.readLong();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcTableMeta.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcTableMeta.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcTableMeta.java
index 5e15189..d4324d6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcTableMeta.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcTableMeta.java
@@ -21,6 +21,7 @@ import java.util.Objects;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -66,13 +67,15 @@ public class JdbcTableMeta implements JdbcRawBinarylizable {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
writer.writeString(schemaName);
writer.writeString(tblName);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
schemaName = reader.readString();
tblName = reader.readString();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java
index 2b51741..d82dcc6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java
@@ -47,7 +47,7 @@ public class OdbcConnectionContext extends ClientListenerAbstractConnectionConte
/** Version 2.3.2: added multiple statements support. */
public static final ClientListenerProtocolVersion VER_2_3_2 = ClientListenerProtocolVersion.create(2, 3, 2);
- /** Version 2.5.0: added authentication and transactions. */
+ /** Version 2.5.0: added authentication. */
public static final ClientListenerProtocolVersion VER_2_5_0 = ClientListenerProtocolVersion.create(2, 5, 0);
/** Version 2.7.0: added precision and scale. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java
index 552841d..3ed6f21 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java
@@ -103,7 +103,7 @@ public class OdbcMessageParser implements ClientListenerMessageParser {
boolean autoCommit = true;
- if (ver.compareTo(OdbcConnectionContext.VER_2_5_0) >= 0)
+ if (ver.compareTo(OdbcConnectionContext.VER_2_7_0) >= 0)
autoCommit = reader.readBoolean();
res = new OdbcQueryExecuteRequest(schema, sql, params, timeout, autoCommit);
@@ -130,7 +130,7 @@ public class OdbcMessageParser implements ClientListenerMessageParser {
boolean autoCommit = true;
- if (ver.compareTo(OdbcConnectionContext.VER_2_5_0) >= 0)
+ if (ver.compareTo(OdbcConnectionContext.VER_2_7_0) >= 0)
autoCommit = reader.readBoolean();
res = new OdbcQueryExecuteBatchRequest(schema, sql, last, params, timeout, autoCommit);
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
index 8672194..0bec66e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
@@ -476,67 +476,67 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
writer.incrementState();
case 2:
- if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
+ if (!writer.writeInt("pageSize", pageSize))
return false;
writer.incrementState();
case 3:
- if (!writer.writeInt("pageSize", pageSize))
+ if (!writer.writeByteArray("paramsBytes", paramsBytes))
return false;
writer.incrementState();
case 4:
- if (!writer.writeByteArray("paramsBytes", paramsBytes))
+ if (!writer.writeMap("parts", parts, MessageCollectionItemType.UUID, MessageCollectionItemType.INT_ARR))
return false;
writer.incrementState();
case 5:
- if (!writer.writeMap("parts", parts, MessageCollectionItemType.UUID, MessageCollectionItemType.INT_ARR))
+ if (!writer.writeCollection("qrys", qrys, MessageCollectionItemType.MSG))
return false;
writer.incrementState();
case 6:
- if (!writer.writeIntArray("qryParts", qryParts))
+ if (!writer.writeLong("reqId", reqId))
return false;
writer.incrementState();
case 7:
- if (!writer.writeCollection("qrys", qrys, MessageCollectionItemType.MSG))
+ if (!writer.writeCollection("tbls", tbls, MessageCollectionItemType.MSG))
return false;
writer.incrementState();
case 8:
- if (!writer.writeLong("reqId", reqId))
+ if (!writer.writeInt("timeout", timeout))
return false;
writer.incrementState();
case 9:
- if (!writer.writeString("schemaName", schemaName))
+ if (!writer.writeMessage("topVer", topVer))
return false;
writer.incrementState();
case 10:
- if (!writer.writeCollection("tbls", tbls, MessageCollectionItemType.MSG))
+ if (!writer.writeIntArray("qryParts", qryParts))
return false;
writer.incrementState();
case 11:
- if (!writer.writeInt("timeout", timeout))
+ if (!writer.writeString("schemaName", schemaName))
return false;
writer.incrementState();
case 12:
- if (!writer.writeMessage("topVer", topVer))
+ if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
return false;
writer.incrementState();
@@ -577,7 +577,7 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
reader.incrementState();
case 2:
- mvccSnapshot = reader.readMessage("mvccSnapshot");
+ pageSize = reader.readInt("pageSize");
if (!reader.isLastRead())
return false;
@@ -585,7 +585,7 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
reader.incrementState();
case 3:
- pageSize = reader.readInt("pageSize");
+ paramsBytes = reader.readByteArray("paramsBytes");
if (!reader.isLastRead())
return false;
@@ -593,7 +593,7 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
reader.incrementState();
case 4:
- paramsBytes = reader.readByteArray("paramsBytes");
+ parts = reader.readMap("parts", MessageCollectionItemType.UUID, MessageCollectionItemType.INT_ARR, false);
if (!reader.isLastRead())
return false;
@@ -601,7 +601,7 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
reader.incrementState();
case 5:
- parts = reader.readMap("parts", MessageCollectionItemType.UUID, MessageCollectionItemType.INT_ARR, false);
+ qrys = reader.readCollection("qrys", MessageCollectionItemType.MSG);
if (!reader.isLastRead())
return false;
@@ -609,7 +609,7 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
reader.incrementState();
case 6:
- qryParts = reader.readIntArray("qryParts");
+ reqId = reader.readLong("reqId");
if (!reader.isLastRead())
return false;
@@ -617,7 +617,7 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
reader.incrementState();
case 7:
- qrys = reader.readCollection("qrys", MessageCollectionItemType.MSG);
+ tbls = reader.readCollection("tbls", MessageCollectionItemType.MSG);
if (!reader.isLastRead())
return false;
@@ -625,7 +625,7 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
reader.incrementState();
case 8:
- reqId = reader.readLong("reqId");
+ timeout = reader.readInt("timeout");
if (!reader.isLastRead())
return false;
@@ -633,7 +633,7 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
reader.incrementState();
case 9:
- schemaName = reader.readString("schemaName");
+ topVer = reader.readMessage("topVer");
if (!reader.isLastRead())
return false;
@@ -641,7 +641,7 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
reader.incrementState();
case 10:
- tbls = reader.readCollection("tbls", MessageCollectionItemType.MSG);
+ qryParts = reader.readIntArray("qryParts");
if (!reader.isLastRead())
return false;
@@ -649,7 +649,7 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
reader.incrementState();
case 11:
- timeout = reader.readInt("timeout");
+ schemaName = reader.readString("schemaName");
if (!reader.isLastRead())
return false;
@@ -657,7 +657,7 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
reader.incrementState();
case 12:
- topVer = reader.readMessage("topVer");
+ mvccSnapshot = reader.readMessage("mvccSnapshot");
if (!reader.isLastRead())
return false;
[24/50] [abbrv] ignite git commit: IGNITE-9387: [ML] Model updating
Posted by ag...@apache.org.
IGNITE-9387: [ML] Model updating
this closes #4659
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f4c18f11
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f4c18f11
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f4c18f11
Branch: refs/heads/ignite-5960
Commit: f4c18f114a435ab818e5c9b179e4854df611dd74
Parents: 02afb43
Author: Alexey Platonov <ap...@gmail.com>
Authored: Tue Sep 4 18:11:48 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Tue Sep 4 18:11:48 2018 +0300
----------------------------------------------------------------------
.../GDBOnTreesClassificationTrainerExample.java | 3 +-
.../GDBOnTreesRegressionTrainerExample.java | 3 +-
.../ml/clustering/kmeans/KMeansTrainer.java | 27 ++-
.../ml/composition/BaggingModelTrainer.java | 20 +++
.../ml/composition/ModelsComposition.java | 10 +-
.../ml/composition/ModelsCompositionFormat.java | 61 +++++++
.../ml/composition/boosting/GDBTrainer.java | 19 ++-
.../ignite/ml/knn/NNClassificationModel.java | 16 ++
.../ml/knn/ann/ANNClassificationModel.java | 15 +-
.../ml/knn/ann/ANNClassificationTrainer.java | 93 +++++++----
.../ignite/ml/knn/ann/ANNModelFormat.java | 12 +-
.../classification/KNNClassificationModel.java | 40 ++++-
.../KNNClassificationTrainer.java | 20 ++-
.../ml/knn/regression/KNNRegressionTrainer.java | 19 ++-
.../ml/math/isolve/lsqr/AbstractLSQR.java | 6 +-
.../ignite/ml/math/isolve/lsqr/LSQROnHeap.java | 2 +-
.../org/apache/ignite/ml/nn/MLPTrainer.java | 25 ++-
.../ml/preprocessing/PreprocessingTrainer.java | 3 +-
.../linear/LinearRegressionLSQRTrainer.java | 27 ++-
.../linear/LinearRegressionSGDTrainer.java | 97 ++++++++++-
.../binomial/LogisticRegressionSGDTrainer.java | 42 ++++-
.../LogRegressionMultiClassModel.java | 9 +
.../LogRegressionMultiClassTrainer.java | 33 +++-
.../SVMLinearBinaryClassificationTrainer.java | 64 +++++--
.../SVMLinearMultiClassClassificationModel.java | 9 +
...VMLinearMultiClassClassificationTrainer.java | 82 +++++++--
.../ignite/ml/trainers/DatasetTrainer.java | 166 +++++++++++++++++++
.../org/apache/ignite/ml/tree/DecisionTree.java | 23 +++
.../RandomForestClassifierTrainer.java | 8 +-
.../tree/randomforest/RandomForestTrainer.java | 28 +++-
.../ignite/ml/clustering/KMeansTrainerTest.java | 82 ++++++---
.../ignite/ml/common/CollectionsTest.java | 9 +-
.../ignite/ml/common/LocalModelsTest.java | 5 +-
.../ml/composition/boosting/GDBTrainerTest.java | 2 +-
.../ignite/ml/knn/ANNClassificationTest.java | 44 +++++
.../ignite/ml/knn/KNNClassificationTest.java | 39 +++++
.../apache/ignite/ml/knn/KNNRegressionTest.java | 40 +++++
.../org/apache/ignite/ml/nn/MLPTrainerTest.java | 64 +++++++
.../linear/LinearRegressionLSQRTrainerTest.java | 51 ++++++
.../linear/LinearRegressionSGDTrainerTest.java | 62 +++++++
.../logistic/LogRegMultiClassTrainerTest.java | 58 +++++++
.../LogisticRegressionSGDTrainerTest.java | 46 +++++
.../ignite/ml/svm/SVMBinaryTrainerTest.java | 41 +++++
.../ignite/ml/svm/SVMMultiClassTrainerTest.java | 43 +++++
.../RandomForestClassifierTrainerTest.java | 33 +++-
.../RandomForestRegressionTrainerTest.java | 31 ++++
46 files changed, 1503 insertions(+), 129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java
index 130b91a..075eab2 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java
@@ -23,6 +23,7 @@ import org.apache.ignite.Ignition;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.ml.Model;
+import org.apache.ignite.ml.composition.ModelsComposition;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.trainers.DatasetTrainer;
@@ -58,7 +59,7 @@ public class GDBOnTreesClassificationTrainerExample {
IgniteCache<Integer, double[]> trainingSet = fillTrainingData(ignite, trainingSetCfg);
// Create regression trainer.
- DatasetTrainer<Model<Vector, Double>, Double> trainer = new GDBBinaryClassifierOnTreesTrainer(1.0, 300, 2, 0.);
+ DatasetTrainer<ModelsComposition, Double> trainer = new GDBBinaryClassifierOnTreesTrainer(1.0, 300, 2, 0.);
// Train decision tree model.
Model<Vector, Double> mdl = trainer.fit(
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java
index 31dd2b0..b2b08d0 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java
@@ -23,6 +23,7 @@ import org.apache.ignite.Ignition;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.ml.Model;
+import org.apache.ignite.ml.composition.ModelsComposition;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.trainers.DatasetTrainer;
@@ -58,7 +59,7 @@ public class GDBOnTreesRegressionTrainerExample {
IgniteCache<Integer, double[]> trainingSet = fillTrainingData(ignite, trainingSetCfg);
// Create regression trainer.
- DatasetTrainer<Model<Vector, Double>, Double> trainer = new GDBRegressionOnTreesTrainer(1.0, 2000, 1, 0.);
+ DatasetTrainer<ModelsComposition, Double> trainer = new GDBRegressionOnTreesTrainer(1.0, 2000, 1, 0.);
// Train decision tree model.
Model<Vector, Double> mdl = trainer.fit(
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
index 5b880fcc..2596dbc 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -72,6 +73,14 @@ public class KMeansTrainer extends SingleLabelDatasetTrainer<KMeansModel> {
*/
@Override public <K, V> KMeansModel fit(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ return updateModel(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected <K, V> KMeansModel updateModel(KMeansModel mdl, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
assert datasetBuilder != null;
PartitionDataBuilder<K, V, EmptyContext, LabeledVectorSet<Double, LabeledVector>> partDataBuilder = new LabeledDatasetPartitionDataBuilderOnHeap<>(
@@ -85,7 +94,7 @@ public class KMeansTrainer extends SingleLabelDatasetTrainer<KMeansModel> {
(upstream, upstreamSize) -> new EmptyContext(),
partDataBuilder
)) {
- final int cols = dataset.compute(org.apache.ignite.ml.structures.Dataset::colSize, (a, b) -> {
+ final Integer cols = dataset.compute(org.apache.ignite.ml.structures.Dataset::colSize, (a, b) -> {
if (a == null)
return b == null ? 0 : b;
if (b == null)
@@ -93,7 +102,12 @@ public class KMeansTrainer extends SingleLabelDatasetTrainer<KMeansModel> {
return b;
});
- centers = initClusterCentersRandomly(dataset, k);
+ if (cols == null)
+ return getLastTrainedModelOrThrowEmptyDatasetException(mdl);
+
+ centers = Optional.ofNullable(mdl)
+ .map(KMeansModel::centers)
+ .orElseGet(() -> initClusterCentersRandomly(dataset, k));
boolean converged = false;
int iteration = 0;
@@ -127,6 +141,11 @@ public class KMeansTrainer extends SingleLabelDatasetTrainer<KMeansModel> {
return new KMeansModel(centers, distance);
}
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(KMeansModel mdl) {
+ return mdl.centers().length == k && mdl.distanceMeasure().equals(distance);
+ }
+
/**
* Prepares the data to define new centroids on current iteration.
*
@@ -281,10 +300,12 @@ public class KMeansTrainer extends SingleLabelDatasetTrainer<KMeansModel> {
return this;
}
+ /**
+ * @return centroid statistics.
+ */
public ConcurrentHashMap<Integer, ConcurrentHashMap<Double, Integer>> getCentroidStat() {
return centroidStat;
}
-
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/composition/BaggingModelTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/BaggingModelTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/BaggingModelTrainer.java
index f439789..493c1da 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/BaggingModelTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/BaggingModelTrainer.java
@@ -177,4 +177,24 @@ public abstract class BaggingModelTrainer extends DatasetTrainer<ModelsCompositi
return VectorUtils.of(newFeaturesValues);
});
}
+
+ /**
+ * Learn new models on dataset and create new Compositions over them and already learned models.
+ *
+ * @param mdl Learned model.
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return New models composition.
+ */
+ @Override public <K, V> ModelsComposition updateModel(ModelsComposition mdl, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ ArrayList<Model<Vector, Double>> newModels = new ArrayList<>(mdl.getModels());
+ newModels.addAll(fit(datasetBuilder, featureExtractor, lbExtractor).getModels());
+
+ return new ModelsComposition(newModels, predictionsAggregator);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsComposition.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsComposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsComposition.java
index e14fa6d..36ee626 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsComposition.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsComposition.java
@@ -19,6 +19,8 @@ package org.apache.ignite.ml.composition;
import java.util.Collections;
import java.util.List;
+import org.apache.ignite.ml.Exportable;
+import org.apache.ignite.ml.Exporter;
import org.apache.ignite.ml.Model;
import org.apache.ignite.ml.composition.predictionsaggregator.PredictionsAggregator;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -27,7 +29,7 @@ import org.apache.ignite.ml.util.ModelTrace;
/**
* Model consisting of several models and prediction aggregation strategy.
*/
-public class ModelsComposition implements Model<Vector, Double> {
+public class ModelsComposition implements Model<Vector, Double>, Exportable<ModelsCompositionFormat> {
/**
* Predictions aggregator.
*/
@@ -78,6 +80,12 @@ public class ModelsComposition implements Model<Vector, Double> {
}
/** {@inheritDoc} */
+ @Override public <P> void saveModel(Exporter<ModelsCompositionFormat, P> exporter, P path) {
+ ModelsCompositionFormat format = new ModelsCompositionFormat(models, predictionsAggregator);
+ exporter.save(format, path);
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return toString(false);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsCompositionFormat.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsCompositionFormat.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsCompositionFormat.java
new file mode 100644
index 0000000..68af0a9
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/ModelsCompositionFormat.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.ignite.ml.Model;
+import org.apache.ignite.ml.composition.predictionsaggregator.PredictionsAggregator;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+
+/**
+ * ModelsComposition representation.
+ *
+ * @see ModelsComposition
+ */
+public class ModelsCompositionFormat implements Serializable {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 9115341364082681837L;
+
+ /** Models. */
+ private List<Model<Vector, Double>> models;
+
+ /** Predictions aggregator. */
+ private PredictionsAggregator predictionsAggregator;
+
+ /**
+ * Creates an instance of ModelsCompositionFormat.
+ *
+ * @param models Models.
+ * @param predictionsAggregator Predictions aggregator.
+ */
+ public ModelsCompositionFormat(List<Model<Vector, Double>> models,PredictionsAggregator predictionsAggregator) {
+ this.models = models;
+ this.predictionsAggregator = predictionsAggregator;
+ }
+
+ /** */
+ public List<Model<Vector, Double>> models() {
+ return models;
+ }
+
+ /** */
+ public PredictionsAggregator predictionsAggregator() {
+ return predictionsAggregator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java
index 5a0f52a..c7f21dd 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java
@@ -52,7 +52,7 @@ import org.jetbrains.annotations.NotNull;
*
* But in practice Decision Trees is most used regressors (see: {@link DecisionTreeRegressionTrainer}).
*/
-public abstract class GDBTrainer extends DatasetTrainer<Model<Vector, Double>, Double> {
+public abstract class GDBTrainer extends DatasetTrainer<ModelsComposition, Double> {
/** Gradient step. */
private final double gradientStep;
@@ -81,7 +81,7 @@ public abstract class GDBTrainer extends DatasetTrainer<Model<Vector, Double>, D
}
/** {@inheritDoc} */
- @Override public <K, V> Model<Vector, Double> fit(DatasetBuilder<K, V> datasetBuilder,
+ @Override public <K, V> ModelsComposition fit(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Vector> featureExtractor,
IgniteBiFunction<K, V, Double> lbExtractor) {
@@ -119,6 +119,21 @@ public abstract class GDBTrainer extends DatasetTrainer<Model<Vector, Double>, D
};
}
+
+ //TODO: This method will be implemented in IGNITE-9412
+ /** {@inheritDoc} */
+ @Override public <K, V> ModelsComposition updateModel(ModelsComposition mdl, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ throw new UnsupportedOperationException();
+ }
+
+ //TODO: This method will be implemented in IGNITE-9412
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(ModelsComposition mdl) {
+ throw new UnsupportedOperationException();
+ }
+
/**
* Defines unique labels in dataset if need (useful in case of classification).
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/knn/NNClassificationModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/NNClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/NNClassificationModel.java
index b7a57f5..d435f91 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/NNClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/NNClassificationModel.java
@@ -174,6 +174,11 @@ public abstract class NNClassificationModel implements Model<Vector, Double>, Ex
return 1.0; // strategy.SIMPLE
}
+ /** */
+ public DistanceMeasure getDistanceMeasure() {
+ return distanceMeasure;
+ }
+
/** {@inheritDoc} */
@Override public int hashCode() {
int res = 1;
@@ -212,6 +217,17 @@ public abstract class NNClassificationModel implements Model<Vector, Double>, Ex
.toString();
}
+ /**
+ * Sets parameters from other model to this model.
+ *
+ * @param mdl Model.
+ */
+ protected void copyParametersFrom(NNClassificationModel mdl) {
+ this.k = mdl.k;
+ this.distanceMeasure = mdl.distanceMeasure;
+ this.stgy = mdl.stgy;
+ }
+
/** */
public abstract <P> void saveModel(Exporter<KNNModelFormat, P> exporter, P path);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java
index e8c0b4a..bec82a9 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java
@@ -44,12 +44,18 @@ public class ANNClassificationModel extends NNClassificationModel {
/** The labeled set of candidates. */
private final LabeledVectorSet<ProbableLabel, LabeledVector> candidates;
+ /** Centroid statistics. */
+ private final ANNClassificationTrainer.CentroidStat centroindsStat;
+
/**
* Build the model based on a candidates set.
* @param centers The candidates set.
+ * @param centroindsStat
*/
- public ANNClassificationModel(LabeledVectorSet<ProbableLabel, LabeledVector> centers) {
+ public ANNClassificationModel(LabeledVectorSet<ProbableLabel, LabeledVector> centers,
+ ANNClassificationTrainer.CentroidStat centroindsStat) {
this.candidates = centers;
+ this.centroindsStat = centroindsStat;
}
/** */
@@ -57,6 +63,11 @@ public class ANNClassificationModel extends NNClassificationModel {
return candidates;
}
+ /** */
+ public ANNClassificationTrainer.CentroidStat getCentroindsStat() {
+ return centroindsStat;
+ }
+
/** {@inheritDoc} */
@Override public Double apply(Vector v) {
List<LabeledVector> neighbors = findKNearestNeighbors(v);
@@ -65,7 +76,7 @@ public class ANNClassificationModel extends NNClassificationModel {
/** */
@Override public <P> void saveModel(Exporter<KNNModelFormat, P> exporter, P path) {
- ANNModelFormat mdlData = new ANNModelFormat(k, distanceMeasure, stgy, candidates);
+ ANNModelFormat mdlData = new ANNModelFormat(k, distanceMeasure, stgy, candidates, centroindsStat);
exporter.save(mdlData, path);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java
index 1c45812..3e32b67 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationTrainer.java
@@ -17,9 +17,13 @@
package org.apache.ignite.ml.knn.ann;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.stream.Collectors;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.ml.clustering.kmeans.KMeansModel;
import org.apache.ignite.ml.clustering.kmeans.KMeansTrainer;
@@ -39,8 +43,8 @@ import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
import org.jetbrains.annotations.NotNull;
/**
- * ANN algorithm trainer to solve multi-class classification task.
- * This trainer is based on ACD strategy and KMeans clustering algorithm to find centroids.
+ * ANN algorithm trainer to solve multi-class classification task. This trainer is based on ACD strategy and KMeans
+ * clustering algorithm to find centroids.
*/
public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClassificationModel> {
/** Amount of clusters. */
@@ -61,29 +65,55 @@ public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClass
/**
* Trains model based on the specified data.
*
- * @param datasetBuilder Dataset builder.
+ * @param datasetBuilder Dataset builder.
* @param featureExtractor Feature extractor.
- * @param lbExtractor Label extractor.
+ * @param lbExtractor Label extractor.
* @return Model.
*/
- @Override public <K, V> ANNClassificationModel fit(DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
- final Vector[] centers = getCentroids(featureExtractor, lbExtractor, datasetBuilder);
+ @Override public <K, V> ANNClassificationModel fit(DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
- final CentroidStat centroidStat = getCentroidStat(datasetBuilder, featureExtractor, lbExtractor, centers);
+ return updateModel(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected <K, V> ANNClassificationModel updateModel(ANNClassificationModel mdl,
+ DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ List<Vector> centers;
+ CentroidStat centroidStat;
+ if (mdl != null) {
+ centers = Arrays.stream(mdl.getCandidates().data()).map(x -> x.features()).collect(Collectors.toList());
+ CentroidStat newStat = getCentroidStat(datasetBuilder, featureExtractor, lbExtractor, centers);
+ if(newStat == null)
+ return mdl;
+ CentroidStat oldStat = mdl.getCentroindsStat();
+ centroidStat = newStat.merge(oldStat);
+ } else {
+ centers = getCentroids(featureExtractor, lbExtractor, datasetBuilder);
+ centroidStat = getCentroidStat(datasetBuilder, featureExtractor, lbExtractor, centers);
+ }
final LabeledVectorSet<ProbableLabel, LabeledVector> dataset = buildLabelsForCandidates(centers, centroidStat);
- return new ANNClassificationModel(dataset);
+ return new ANNClassificationModel(dataset, centroidStat);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(ANNClassificationModel mdl) {
+ return mdl.getDistanceMeasure().equals(distance) && mdl.getCandidates().rowSize() == k;
}
/** */
- @NotNull private LabeledVectorSet<ProbableLabel, LabeledVector> buildLabelsForCandidates(Vector[] centers, CentroidStat centroidStat) {
+ @NotNull private LabeledVectorSet<ProbableLabel, LabeledVector> buildLabelsForCandidates(List<Vector> centers,
+ CentroidStat centroidStat) {
// init
- final LabeledVector<Vector, ProbableLabel>[] arr = new LabeledVector[centers.length];
+ final LabeledVector<Vector, ProbableLabel>[] arr = new LabeledVector[centers.size()];
// fill label for each centroid
- for (int i = 0; i < centers.length; i++)
- arr[i] = new LabeledVector<>(centers[i], fillProbableLabel(i, centroidStat));
+ for (int i = 0; i < centers.size(); i++)
+ arr[i] = new LabeledVector<>(centers.get(i), fillProbableLabel(i, centroidStat));
return new LabeledVectorSet<>(arr);
}
@@ -92,13 +122,14 @@ public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClass
* Perform KMeans clusterization algorithm to find centroids.
*
* @param featureExtractor Feature extractor.
- * @param lbExtractor Label extractor.
- * @param datasetBuilder The dataset builder.
- * @param <K> Type of a key in {@code upstream} data.
- * @param <V> Type of a value in {@code upstream} data.
+ * @param lbExtractor Label extractor.
+ * @param datasetBuilder The dataset builder.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
* @return The arrays of vectors.
*/
- private <K, V> Vector[] getCentroids(IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor, DatasetBuilder<K, V> datasetBuilder) {
+ private <K, V> List<Vector> getCentroids(IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor, DatasetBuilder<K, V> datasetBuilder) {
KMeansTrainer trainer = new KMeansTrainer()
.withK(k)
.withMaxIterations(maxIterations)
@@ -112,7 +143,7 @@ public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClass
lbExtractor
);
- return mdl.centers();
+ return Arrays.asList(mdl.centers());
}
/** */
@@ -125,21 +156,24 @@ public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClass
ConcurrentHashMap<Double, Integer> centroidLbDistribution
= centroidStat.centroidStat().get(centroidIdx);
- if(centroidStat.counts.containsKey(centroidIdx)){
+ if (centroidStat.counts.containsKey(centroidIdx)) {
int clusterSize = centroidStat
.counts
.get(centroidIdx);
clsLbls.keySet().forEach(
- (label) -> clsLbls.put(label, centroidLbDistribution.containsKey(label) ? ((double) (centroidLbDistribution.get(label)) / clusterSize) : 0.0)
+ (label) -> clsLbls.put(label, centroidLbDistribution.containsKey(label) ? ((double)(centroidLbDistribution.get(label)) / clusterSize) : 0.0)
);
}
return new ProbableLabel(clsLbls);
}
/** */
- private <K, V> CentroidStat getCentroidStat(DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor, Vector[] centers) {
+ private <K, V> CentroidStat getCentroidStat(DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor, List<Vector> centers) {
+
PartitionDataBuilder<K, V, EmptyContext, LabeledVectorSet<Double, LabeledVector>> partDataBuilder = new LabeledDatasetPartitionDataBuilderOnHeap<>(
featureExtractor,
lbExtractor
@@ -174,7 +208,7 @@ public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClass
}
res.counts.merge(centroidIdx, 1,
- (IgniteBiFunction<Integer, Integer, Integer>) (i1, i2) -> i1 + i2);
+ (IgniteBiFunction<Integer, Integer, Integer>)(i1, i2) -> i1 + i2);
}
return res;
}, (a, b) -> {
@@ -194,15 +228,15 @@ public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClass
* Find the closest cluster center index and distance to it from a given point.
*
* @param centers Centers to look in.
- * @param pnt Point.
+ * @param pnt Point.
*/
- private IgniteBiTuple<Integer, Double> findClosestCentroid(Vector[] centers, LabeledVector pnt) {
+ private IgniteBiTuple<Integer, Double> findClosestCentroid(List<Vector> centers, LabeledVector pnt) {
double bestDistance = Double.POSITIVE_INFINITY;
int bestInd = 0;
- for (int i = 0; i < centers.length; i++) {
- if (centers[i] != null) {
- double dist = distance.compute(centers[i], pnt.features());
+ for (int i = 0; i < centers.size(); i++) {
+ if (centers.get(i) != null) {
+ double dist = distance.compute(centers.get(i), pnt.features());
if (dist < bestDistance) {
bestDistance = dist;
bestInd = i;
@@ -212,7 +246,6 @@ public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClass
return new IgniteBiTuple<>(bestInd, bestDistance);
}
-
/**
* Gets the amount of clusters.
*
@@ -314,7 +347,9 @@ public class ANNClassificationTrainer extends SingleLabelDatasetTrainer<ANNClass
}
/** Service class used for statistics. */
- public static class CentroidStat {
+ public static class CentroidStat implements Serializable {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 7624883170532045144L;
/** Count of points closest to the center with a given index. */
ConcurrentHashMap<Integer, ConcurrentHashMap<Double, Integer>> centroidStat = new ConcurrentHashMap<>();
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNModelFormat.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNModelFormat.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNModelFormat.java
index e10f3b2..be09828 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNModelFormat.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNModelFormat.java
@@ -30,6 +30,9 @@ import org.apache.ignite.ml.structures.LabeledVectorSet;
* @see ANNClassificationModel
*/
public class ANNModelFormat extends KNNModelFormat implements Serializable {
+ /** Centroid statistics. */
+ private final ANNClassificationTrainer.CentroidStat candidatesStat;
+
/** The labeled set of candidates. */
private LabeledVectorSet<ProbableLabel, LabeledVector> candidates;
@@ -38,15 +41,18 @@ public class ANNModelFormat extends KNNModelFormat implements Serializable {
* @param k Amount of nearest neighbors.
* @param measure Distance measure.
* @param stgy kNN strategy.
+ * @param candidatesStat
*/
public ANNModelFormat(int k,
- DistanceMeasure measure,
- NNStrategy stgy,
- LabeledVectorSet<ProbableLabel, LabeledVector> candidates) {
+ DistanceMeasure measure,
+ NNStrategy stgy,
+ LabeledVectorSet<ProbableLabel, LabeledVector> candidates,
+ ANNClassificationTrainer.CentroidStat candidatesStat) {
this.k = k;
this.distanceMeasure = measure;
this.stgy = stgy;
this.candidates = candidates;
+ this.candidatesStat = candidatesStat;
}
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java
index 0b88f81..0d03ee5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ml.knn.classification;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -42,25 +43,29 @@ public class KNNClassificationModel extends NNClassificationModel implements Exp
/** */
private static final long serialVersionUID = -127386523291350345L;
- /** Dataset. */
- private Dataset<EmptyContext, LabeledVectorSet<Double, LabeledVector>> dataset;
+ /** Datasets. */
+ private List<Dataset<EmptyContext, LabeledVectorSet<Double, LabeledVector>>> datasets;
/**
* Builds the model via prepared dataset.
+ *
* @param dataset Specially prepared object to run algorithm over it.
*/
public KNNClassificationModel(Dataset<EmptyContext, LabeledVectorSet<Double, LabeledVector>> dataset) {
- this.dataset = dataset;
+ this.datasets = new ArrayList<>();
+ if (dataset != null)
+ datasets.add(dataset);
}
/** {@inheritDoc} */
@Override public Double apply(Vector v) {
- if(dataset != null) {
+ if (!datasets.isEmpty()) {
List<LabeledVector> neighbors = findKNearestNeighbors(v);
return classify(neighbors, v, stgy);
- } else
+ } else {
throw new IllegalStateException("The train kNN dataset is null");
+ }
}
/** */
@@ -77,6 +82,17 @@ public class KNNClassificationModel extends NNClassificationModel implements Exp
* @return K-nearest neighbors.
*/
protected List<LabeledVector> findKNearestNeighbors(Vector v) {
+ List<LabeledVector> neighborsFromPartitions = datasets.stream()
+ .flatMap(dataset -> findKNearestNeighborsInDataset(v, dataset).stream())
+ .collect(Collectors.toList());
+
+ LabeledVectorSet<Double, LabeledVector> neighborsToFilter = buildLabeledDatasetOnListOfVectors(neighborsFromPartitions);
+
+ return Arrays.asList(getKClosestVectors(neighborsToFilter, getDistances(v, neighborsToFilter)));
+ }
+
+ private List<LabeledVector> findKNearestNeighborsInDataset(Vector v,
+ Dataset<EmptyContext, LabeledVectorSet<Double, LabeledVector>> dataset) {
List<LabeledVector> neighborsFromPartitions = dataset.compute(data -> {
TreeMap<Double, Set<Integer>> distanceIdxPairs = getDistances(v, data);
return Arrays.asList(getKClosestVectors(data, distanceIdxPairs));
@@ -88,12 +104,14 @@ public class KNNClassificationModel extends NNClassificationModel implements Exp
return Stream.concat(a.stream(), b.stream()).collect(Collectors.toList());
});
+ if(neighborsFromPartitions == null)
+ return Collections.emptyList();
+
LabeledVectorSet<Double, LabeledVector> neighborsToFilter = buildLabeledDatasetOnListOfVectors(neighborsFromPartitions);
return Arrays.asList(getKClosestVectors(neighborsToFilter, getDistances(v, neighborsToFilter)));
}
-
/** */
private double classify(List<LabeledVector> neighbors, Vector v, NNStrategy stgy) {
Map<Double, Double> clsVotes = new HashMap<>();
@@ -116,5 +134,13 @@ public class KNNClassificationModel extends NNClassificationModel implements Exp
return getClassWithMaxVotes(clsVotes);
}
-
+ /**
+ * Copy parameters from other model and save all datasets from it.
+ *
+ * @param model Model.
+ */
+ public void copyStateFrom(KNNClassificationModel model) {
+ this.copyParametersFrom(model);
+ datasets.addAll(model.datasets);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java
index e0a81f9..1a3ff73 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java
@@ -37,6 +37,24 @@ public class KNNClassificationTrainer extends SingleLabelDatasetTrainer<KNNClass
*/
@Override public <K, V> KNNClassificationModel fit(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
- return new KNNClassificationModel(KNNUtils.buildDataset(datasetBuilder, featureExtractor, lbExtractor));
+
+ return updateModel(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> KNNClassificationModel updateModel(KNNClassificationModel mdl,
+ DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ KNNClassificationModel res = new KNNClassificationModel(KNNUtils.buildDataset(datasetBuilder,
+ featureExtractor, lbExtractor));
+ if (mdl != null)
+ res.copyStateFrom(mdl);
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(KNNClassificationModel mdl) {
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java
index 395ce61..7a42dc8 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java
@@ -37,6 +37,23 @@ public class KNNRegressionTrainer extends SingleLabelDatasetTrainer<KNNRegressio
*/
public <K, V> KNNRegressionModel fit(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
- return new KNNRegressionModel(KNNUtils.buildDataset(datasetBuilder, featureExtractor, lbExtractor));
+
+ return updateModel(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> KNNRegressionModel updateModel(KNNRegressionModel mdl, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ KNNRegressionModel res = new KNNRegressionModel(KNNUtils.buildDataset(datasetBuilder,
+ featureExtractor, lbExtractor));
+ if (mdl != null)
+ res.copyStateFrom(mdl);
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(KNNRegressionModel mdl) {
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/AbstractLSQR.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/AbstractLSQR.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/AbstractLSQR.java
index 7a362f7..c9281c0 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/AbstractLSQR.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/AbstractLSQR.java
@@ -78,7 +78,9 @@ public abstract class AbstractLSQR {
*/
public LSQRResult solve(double damp, double atol, double btol, double conlim, double iterLim, boolean calcVar,
double[] x0) {
- int n = getColumns();
+ Integer n = getColumns();
+ if(n == null)
+ return null;
if (iterLim < 0)
iterLim = 2 * n;
@@ -313,7 +315,7 @@ public abstract class AbstractLSQR {
protected abstract double[] iter(double bnorm, double[] target);
/** */
- protected abstract int getColumns();
+ protected abstract Integer getColumns();
/** */
private static double[] symOrtho(double a, double b) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java
index f75caef..14356e1 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java
@@ -100,7 +100,7 @@ public class LSQROnHeap<K, V> extends AbstractLSQR implements AutoCloseable {
*
* @return number of columns
*/
- @Override protected int getColumns() {
+ @Override protected Integer getColumns() {
return dataset.compute(
data -> data.getFeatures() == null ? null : data.getFeatures().length / data.getRows(),
(a, b) -> {
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
index 6727ba9..8f1a4cb 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
@@ -111,12 +111,25 @@ public class MLPTrainer<P extends Serializable> extends MultiLabelDatasetTrainer
public <K, V> MultilayerPerceptron fit(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, double[]> lbExtractor) {
+ return updateModel(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected <K, V> MultilayerPerceptron updateModel(MultilayerPerceptron lastLearnedModel,
+ DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, double[]> lbExtractor) {
+
try (Dataset<EmptyContext, SimpleLabeledDatasetData> dataset = datasetBuilder.build(
new EmptyContextBuilder<>(),
new SimpleLabeledDatasetDataBuilder<>(featureExtractor, lbExtractor)
)) {
- MLPArchitecture arch = archSupplier.apply(dataset);
- MultilayerPerceptron mdl = new MultilayerPerceptron(arch, new RandomInitializer(seed));
+ MultilayerPerceptron mdl;
+ if (lastLearnedModel != null) {
+ mdl = lastLearnedModel;
+ } else {
+ MLPArchitecture arch = archSupplier.apply(dataset);
+ mdl = new MultilayerPerceptron(arch, new RandomInitializer(seed));
+ }
ParameterUpdateCalculator<? super MultilayerPerceptron, P> updater = updatesStgy.getUpdatesCalculator();
for (int i = 0; i < maxIterations; i += locIterations) {
@@ -178,6 +191,9 @@ public class MLPTrainer<P extends Serializable> extends MultiLabelDatasetTrainer
}
);
+ if (totUp == null)
+ return getLastTrainedModelOrThrowEmptyDatasetException(lastLearnedModel);
+
P update = updatesStgy.allUpdatesReducer().apply(totUp);
mdl = updater.update(mdl, update);
}
@@ -189,6 +205,11 @@ public class MLPTrainer<P extends Serializable> extends MultiLabelDatasetTrainer
}
}
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(MultilayerPerceptron mdl) {
+ return true;
+ }
+
/**
* Builds a batch of the data by fetching specified rows.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
index 1886ee5..b977864 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ml.preprocessing;
+import java.util.Map;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.ml.dataset.DatasetBuilder;
@@ -24,8 +25,6 @@ import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import java.util.Map;
-
/**
* Trainer for preprocessor.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
index 8197779..5497177 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
@@ -38,16 +38,34 @@ public class LinearRegressionLSQRTrainer extends SingleLabelDatasetTrainer<Linea
@Override public <K, V> LinearRegressionModel fit(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+ return updateModel(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected <K, V> LinearRegressionModel updateModel(LinearRegressionModel mdl,
+ DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
LSQRResult res;
try (LSQROnHeap<K, V> lsqr = new LSQROnHeap<>(
datasetBuilder,
new SimpleLabeledDatasetDataBuilder<>(
new FeatureExtractorWrapper<>(featureExtractor),
- lbExtractor.andThen(e -> new double[]{e})
+ lbExtractor.andThen(e -> new double[] {e})
)
)) {
- res = lsqr.solve(0, 1e-12, 1e-12, 1e8, -1, false, null);
+ double[] x0 = null;
+ if (mdl != null) {
+ int x0Size = mdl.getWeights().size() + 1;
+ Vector weights = mdl.getWeights().like(x0Size);
+ mdl.getWeights().nonZeroes().forEach(ith -> weights.set(ith.index(), ith.get()));
+ weights.set(weights.size() - 1, mdl.getIntercept());
+ x0 = weights.asArray();
+ }
+ res = lsqr.solve(0, 1e-12, 1e-12, 1e8, -1, false, x0);
+ if (res == null)
+ return getLastTrainedModelOrThrowEmptyDatasetException(mdl);
}
catch (Exception e) {
throw new RuntimeException(e);
@@ -58,4 +76,9 @@ public class LinearRegressionLSQRTrainer extends SingleLabelDatasetTrainer<Linea
return new LinearRegressionModel(weights, x[x.length - 1]);
}
+
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(LinearRegressionModel mdl) {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
index 44f60d1..125ed24 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
@@ -19,6 +19,7 @@ package org.apache.ignite.ml.regressions.linear;
import java.io.Serializable;
import java.util.Arrays;
+import java.util.Optional;
import org.apache.ignite.ml.dataset.Dataset;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
@@ -34,6 +35,7 @@ import org.apache.ignite.ml.nn.UpdatesStrategy;
import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
import org.apache.ignite.ml.optimization.LossFunctions;
import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
+import org.jetbrains.annotations.NotNull;
/**
* Trainer of the linear regression model based on stochastic gradient descent algorithm.
@@ -43,16 +45,16 @@ public class LinearRegressionSGDTrainer<P extends Serializable> extends SingleLa
private final UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy;
/** Max number of iteration. */
- private final int maxIterations;
+ private int maxIterations = 1000;
/** Batch size. */
- private final int batchSize;
+ private int batchSize = 10;
/** Number of local iterations. */
- private final int locIterations;
+ private int locIterations = 100;
/** Seed for random generator. */
- private final long seed;
+ private long seed = System.currentTimeMillis();
/**
* Constructs a new instance of linear regression SGD trainer.
@@ -72,10 +74,24 @@ public class LinearRegressionSGDTrainer<P extends Serializable> extends SingleLa
this.seed = seed;
}
+ /**
+ * Constructs a new instance of linear regression SGD trainer.
+ */
+ public LinearRegressionSGDTrainer(UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy) {
+ this.updatesStgy = updatesStgy;
+ }
+
/** {@inheritDoc} */
@Override public <K, V> LinearRegressionModel fit(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+ return updateModel(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected <K, V> LinearRegressionModel updateModel(LinearRegressionModel mdl, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier = dataset -> {
int cols = dataset.compute(data -> {
@@ -108,7 +124,10 @@ public class LinearRegressionSGDTrainer<P extends Serializable> extends SingleLa
IgniteBiFunction<K, V, double[]> lbE = (IgniteBiFunction<K, V, double[]>)(k, v) -> new double[] {lbExtractor.apply(k, v)};
- MultilayerPerceptron mlp = trainer.fit(datasetBuilder, featureExtractor, lbE);
+ MultilayerPerceptron mlp = Optional.ofNullable(mdl)
+ .map(this::restoreMLPState)
+ .map(m -> trainer.update(m, datasetBuilder, featureExtractor, lbE))
+ .orElseGet(() -> trainer.fit(datasetBuilder, featureExtractor, lbE));
double[] p = mlp.parameters().getStorage().data();
@@ -117,4 +136,72 @@ public class LinearRegressionSGDTrainer<P extends Serializable> extends SingleLa
p[p.length - 1]
);
}
+
+ /**
+ * @param mdl Model.
+ * @return state of MLP from last learning.
+ */
+ @NotNull private MultilayerPerceptron restoreMLPState(LinearRegressionModel mdl) {
+ Vector weights = mdl.getWeights();
+ double intercept = mdl.getIntercept();
+ MLPArchitecture architecture1 = new MLPArchitecture(weights.size());
+ architecture1 = architecture1.withAddedLayer(1, true, Activators.LINEAR);
+ MLPArchitecture architecture = architecture1;
+ MultilayerPerceptron perceptron = new MultilayerPerceptron(architecture);
+
+ Vector mlpState = weights.like(weights.size() + 1);
+ weights.nonZeroes().forEach(ith -> mlpState.set(ith.index(), ith.get()));
+ mlpState.set(mlpState.size() - 1, intercept);
+ perceptron.setParameters(mlpState);
+ return perceptron;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(LinearRegressionModel mdl) {
+ return true;
+ }
+
+ /**
+ * Set up the max number of iterations before convergence.
+ *
+ * @param maxIterations The parameter value.
+ * @return Model with new max number of iterations before convergence parameter value.
+ */
+ public LinearRegressionSGDTrainer<P> withMaxIterations(int maxIterations) {
+ this.maxIterations = maxIterations;
+ return this;
+ }
+
+ /**
+ * Set up the batchSize parameter.
+ *
+ * @param batchSize The size of learning batch.
+ * @return Trainer with new batch size parameter value.
+ */
+ public LinearRegressionSGDTrainer<P> withBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ return this;
+ }
+
+ /**
+ * Set up the amount of local iterations of SGD algorithm.
+ *
+ * @param amountOfLocIterations The parameter value.
+ * @return Trainer with new locIterations parameter value.
+ */
+ public LinearRegressionSGDTrainer<P> withLocIterations(int amountOfLocIterations) {
+ this.locIterations = amountOfLocIterations;
+ return this;
+ }
+
+ /**
+ * Set up the random seed parameter.
+ *
+ * @param seed Seed for random generator.
+ * @return Trainer with new seed parameter value.
+ */
+ public LinearRegressionSGDTrainer<P> withSeed(long seed) {
+ this.seed = seed;
+ return this;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java
index 6396279..839dab5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java
@@ -34,6 +34,7 @@ import org.apache.ignite.ml.nn.UpdatesStrategy;
import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
import org.apache.ignite.ml.optimization.LossFunctions;
import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
+import org.jetbrains.annotations.NotNull;
/**
* Trainer of the logistic regression model based on stochastic gradient descent algorithm.
@@ -76,8 +77,15 @@ public class LogisticRegressionSGDTrainer<P extends Serializable> extends Single
@Override public <K, V> LogisticRegressionModel fit(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
- IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier = dataset -> {
+ return updateModel(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected <K, V> LogisticRegressionModel updateModel(LogisticRegressionModel mdl,
+ DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+ IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier = dataset -> {
int cols = dataset.compute(data -> {
if (data.getFeatures() == null)
return null;
@@ -106,7 +114,13 @@ public class LogisticRegressionSGDTrainer<P extends Serializable> extends Single
seed
);
- MultilayerPerceptron mlp = trainer.fit(datasetBuilder, featureExtractor, (k, v) -> new double[] {lbExtractor.apply(k, v)});
+ IgniteBiFunction<K, V, double[]> lbExtractorWrapper = (k, v) -> new double[] {lbExtractor.apply(k, v)};
+ MultilayerPerceptron mlp;
+ if(mdl != null) {
+ mlp = restoreMLPState(mdl);
+ mlp = trainer.update(mlp, datasetBuilder, featureExtractor, lbExtractorWrapper);
+ } else
+ mlp = trainer.fit(datasetBuilder, featureExtractor, lbExtractorWrapper);
double[] params = mlp.parameters().getStorage().data();
@@ -114,4 +128,28 @@ public class LogisticRegressionSGDTrainer<P extends Serializable> extends Single
params[params.length - 1]
);
}
+
+ /**
+ * @param mdl Model.
+ * @return state of MLP from last learning.
+ */
+ @NotNull private MultilayerPerceptron restoreMLPState(LogisticRegressionModel mdl) {
+ Vector weights = mdl.weights();
+ double intercept = mdl.intercept();
+ MLPArchitecture architecture1 = new MLPArchitecture(weights.size());
+ architecture1 = architecture1.withAddedLayer(1, true, Activators.SIGMOID);
+ MLPArchitecture architecture = architecture1;
+ MultilayerPerceptron perceptron = new MultilayerPerceptron(architecture);
+
+ Vector mlpState = weights.like(weights.size() + 1);
+ weights.nonZeroes().forEach(ith -> mlpState.set(ith.index(), ith.get()));
+ mlpState.set(mlpState.size() - 1, intercept);
+ perceptron.setParameters(mlpState);
+ return perceptron;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(LogisticRegressionModel mdl) {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassModel.java
index 56d2d29..a7c9118 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassModel.java
@@ -21,6 +21,7 @@ import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.TreeMap;
import org.apache.ignite.ml.Exportable;
import org.apache.ignite.ml.Exporter;
@@ -103,4 +104,12 @@ public class LogRegressionMultiClassModel implements Model<Vector, Double>, Expo
public void add(double clsLb, LogisticRegressionModel mdl) {
models.put(clsLb, mdl);
}
+
+ /**
+ * @param clsLb Class label.
+ * @return model for class label if it exists.
+ */
+ public Optional<LogisticRegressionModel> getModel(Double clsLb) {
+ return Optional.ofNullable(models.get(clsLb));
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java
index 4885373..eb44301 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -33,6 +34,7 @@ import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.nn.MultilayerPerceptron;
import org.apache.ignite.ml.nn.UpdatesStrategy;
+import org.apache.ignite.ml.regressions.logistic.binomial.LogisticRegressionModel;
import org.apache.ignite.ml.regressions.logistic.binomial.LogisticRegressionSGDTrainer;
import org.apache.ignite.ml.structures.partition.LabelPartitionDataBuilderOnHeap;
import org.apache.ignite.ml.structures.partition.LabelPartitionDataOnHeap;
@@ -71,6 +73,19 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
IgniteBiFunction<K, V, Double> lbExtractor) {
List<Double> classes = extractClassLabels(datasetBuilder, lbExtractor);
+ return updateModel(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> LogRegressionMultiClassModel updateModel(LogRegressionMultiClassModel mdl,
+ DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ List<Double> classes = extractClassLabels(datasetBuilder, lbExtractor);
+
+ if(classes.isEmpty())
+ return getLastTrainedModelOrThrowEmptyDatasetException(mdl);
+
LogRegressionMultiClassModel multiClsMdl = new LogRegressionMultiClassModel();
classes.forEach(clsLb -> {
@@ -85,12 +100,23 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
else
return 0.0;
};
- multiClsMdl.add(clsLb, trainer.fit(datasetBuilder, featureExtractor, lbTransformer));
+
+ LogisticRegressionModel model = Optional.ofNullable(mdl)
+ .flatMap(multiClassModel -> multiClassModel.getModel(clsLb))
+ .map(learnedModel -> trainer.update(learnedModel, datasetBuilder, featureExtractor, lbTransformer))
+ .orElseGet(() -> trainer.fit(datasetBuilder, featureExtractor, lbTransformer));
+
+ multiClsMdl.add(clsLb, model);
});
return multiClsMdl;
}
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(LogRegressionMultiClassModel mdl) {
+ return true;
+ }
+
/** Iterates among dataset and collects class labels. */
private <K, V> List<Double> extractClassLabels(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Double> lbExtractor) {
@@ -121,7 +147,8 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
return Stream.of(a, b).flatMap(Collection::stream).collect(Collectors.toSet());
});
- res.addAll(clsLabels);
+ if (clsLabels != null)
+ res.addAll(clsLabels);
}
catch (Exception e) {
@@ -191,7 +218,7 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
}
/**
- * Set up the regularization parameter.
+ * Set up the random seed parameter.
*
* @param seed Seed for random generator.
* @return Trainer with new seed parameter value.
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
index 933a712..573df1a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
@@ -22,9 +22,11 @@ import org.apache.ignite.ml.dataset.Dataset;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.PartitionDataBuilder;
import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.math.StorageConstants;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.impl.DenseVector;
+import org.apache.ignite.ml.math.primitives.vector.impl.SparseVector;
import org.apache.ignite.ml.structures.LabeledVector;
import org.apache.ignite.ml.structures.LabeledVectorSet;
import org.apache.ignite.ml.structures.partition.LabeledDatasetPartitionDataBuilderOnHeap;
@@ -61,6 +63,14 @@ public class SVMLinearBinaryClassificationTrainer extends SingleLabelDatasetTrai
@Override public <K, V> SVMLinearBinaryClassificationModel fit(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+ return updateModel(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected <K, V> SVMLinearBinaryClassificationModel updateModel(SVMLinearBinaryClassificationModel mdl,
+ DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+
assert datasetBuilder != null;
PartitionDataBuilder<K, V, EmptyContext, LabeledVectorSet<Double, LabeledVector>> partDataBuilder = new LabeledDatasetPartitionDataBuilderOnHeap<>(
@@ -74,29 +84,57 @@ public class SVMLinearBinaryClassificationTrainer extends SingleLabelDatasetTrai
(upstream, upstreamSize) -> new EmptyContext(),
partDataBuilder
)) {
- final int cols = dataset.compute(org.apache.ignite.ml.structures.Dataset::colSize, (a, b) -> {
- if (a == null)
- return b == null ? 0 : b;
- if (b == null)
- return a;
- return b;
- });
-
- final int weightVectorSizeWithIntercept = cols + 1;
-
- weights = initializeWeightsWithZeros(weightVectorSizeWithIntercept);
+ if (mdl == null) {
+ final int cols = dataset.compute(org.apache.ignite.ml.structures.Dataset::colSize, (a, b) -> {
+ if (a == null)
+ return b == null ? 0 : b;
+ if (b == null)
+ return a;
+ return b;
+ });
+
+ final int weightVectorSizeWithIntercept = cols + 1;
+ weights = initializeWeightsWithZeros(weightVectorSizeWithIntercept);
+ } else {
+ weights = getStateVector(mdl);
+ }
for (int i = 0; i < this.getAmountOfIterations(); i++) {
Vector deltaWeights = calculateUpdates(weights, dataset);
+ if (deltaWeights == null)
+ return getLastTrainedModelOrThrowEmptyDatasetException(mdl);
+
weights = weights.plus(deltaWeights); // creates new vector
}
- }
- catch (Exception e) {
+ } catch (Exception e) {
throw new RuntimeException(e);
}
return new SVMLinearBinaryClassificationModel(weights.viewPart(1, weights.size() - 1), weights.get(0));
}
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(SVMLinearBinaryClassificationModel mdl) {
+ return true;
+ }
+
+ /**
+ * @param mdl Model.
+ * @return vector of model weights with intercept.
+ */
+ private Vector getStateVector(SVMLinearBinaryClassificationModel mdl) {
+ double intercept = mdl.intercept();
+ Vector weights = mdl.weights();
+
+ int stateVectorSize = weights.size() + 1;
+ Vector result = weights.isDense() ?
+ new DenseVector(stateVectorSize) :
+ new SparseVector(stateVectorSize, StorageConstants.RANDOM_ACCESS_MODE);
+
+ result.set(0, intercept);
+ weights.nonZeroes().forEach(ith -> result.set(ith.index(), ith.get()));
+ return result;
+ }
+
/** */
@NotNull private Vector initializeWeightsWithZeros(int vectorSize) {
return new DenseVector(vectorSize);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
index 4b04824..46bf4b2 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
@@ -21,6 +21,7 @@ import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.TreeMap;
import org.apache.ignite.ml.Exportable;
import org.apache.ignite.ml.Exporter;
@@ -102,4 +103,12 @@ public class SVMLinearMultiClassClassificationModel implements Model<Vector, Dou
public void add(double clsLb, SVMLinearBinaryClassificationModel mdl) {
models.put(clsLb, mdl);
}
+
+ /**
+ * @param clsLb Class label.
+ * @return model trained for target class if it exists.
+ */
+ public Optional<SVMLinearBinaryClassificationModel> getModelForClass(double clsLb) {
+ return Optional.of(models.get(clsLb));
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
index 4b7cc95..b77baa2 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
@@ -57,15 +57,26 @@ public class SVMLinearMultiClassClassificationTrainer
/**
* Trains model based on the specified data.
*
- * @param datasetBuilder Dataset builder.
+ * @param datasetBuilder Dataset builder.
* @param featureExtractor Feature extractor.
- * @param lbExtractor Label extractor.
+ * @param lbExtractor Label extractor.
* @return Model.
*/
@Override public <K, V> SVMLinearMultiClassClassificationModel fit(DatasetBuilder<K, V> datasetBuilder,
- IgniteBiFunction<K, V, Vector> featureExtractor,
- IgniteBiFunction<K, V, Double> lbExtractor) {
+ IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+ return updateModel(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> SVMLinearMultiClassClassificationModel updateModel(
+ SVMLinearMultiClassClassificationModel mdl,
+ DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+
List<Double> classes = extractClassLabels(datasetBuilder, lbExtractor);
+ if (classes.isEmpty())
+ return getLastTrainedModelOrThrowEmptyDatasetException(mdl);
SVMLinearMultiClassClassificationModel multiClsMdl = new SVMLinearMultiClassClassificationModel();
@@ -84,14 +95,60 @@ public class SVMLinearMultiClassClassificationTrainer
else
return -1.0;
};
- multiClsMdl.add(clsLb, trainer.fit(datasetBuilder, featureExtractor, lbTransformer));
+
+ SVMLinearBinaryClassificationModel model;
+ if (mdl == null)
+ model = learnNewModel(trainer, datasetBuilder, featureExtractor, lbTransformer);
+ else
+ model = updateModel(mdl, clsLb, trainer, datasetBuilder, featureExtractor, lbTransformer);
+ multiClsMdl.add(clsLb, model);
});
return multiClsMdl;
}
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(SVMLinearMultiClassClassificationModel mdl) {
+ return true;
+ }
+
+ /**
+ * Trains model based on the specified data.
+ *
+ * @param svmTrainer Prepared SVM trainer.
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ */
+ private <K, V> SVMLinearBinaryClassificationModel learnNewModel(SVMLinearBinaryClassificationTrainer svmTrainer,
+ DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ return svmTrainer.fit(datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /**
+ * Updates already learned model or fit new model if there is no model for current class label.
+ *
+ * @param multiClsMdl Learning multi-class model.
+ * @param clsLb Current class label.
+ * @param svmTrainer Prepared SVM trainer.
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ */
+ private <K, V> SVMLinearBinaryClassificationModel updateModel(SVMLinearMultiClassClassificationModel multiClsMdl,
+ Double clsLb, SVMLinearBinaryClassificationTrainer svmTrainer, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ return multiClsMdl.getModelForClass(clsLb)
+ .map(learnedModel -> svmTrainer.update(learnedModel, datasetBuilder, featureExtractor, lbExtractor))
+ .orElseGet(() -> svmTrainer.fit(datasetBuilder, featureExtractor, lbExtractor));
+ }
+
/** Iterates among dataset and collects class labels. */
- private <K, V> List<Double> extractClassLabels(DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Double> lbExtractor) {
+ private <K, V> List<Double> extractClassLabels(DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
assert datasetBuilder != null;
PartitionDataBuilder<K, V, EmptyContext, LabelPartitionDataOnHeap> partDataBuilder = new LabelPartitionDataBuilderOnHeap<>(lbExtractor);
@@ -107,7 +164,8 @@ public class SVMLinearMultiClassClassificationTrainer
final double[] lbs = data.getY();
- for (double lb : lbs) locClsLabels.add(lb);
+ for (double lb : lbs)
+ locClsLabels.add(lb);
return locClsLabels;
}, (a, b) -> {
@@ -118,8 +176,8 @@ public class SVMLinearMultiClassClassificationTrainer
return Stream.of(a, b).flatMap(Collection::stream).collect(Collectors.toSet());
});
- res.addAll(clsLabels);
-
+ if (clsLabels != null)
+ res.addAll(clsLabels);
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -132,7 +190,7 @@ public class SVMLinearMultiClassClassificationTrainer
* @param lambda The regularization parameter. Should be more than 0.0.
* @return Trainer with new lambda parameter value.
*/
- public SVMLinearMultiClassClassificationTrainer withLambda(double lambda) {
+ public SVMLinearMultiClassClassificationTrainer withLambda(double lambda) {
assert lambda > 0.0;
this.lambda = lambda;
return this;
@@ -162,7 +220,7 @@ public class SVMLinearMultiClassClassificationTrainer
* @param amountOfIterations The parameter value.
* @return Trainer with new amountOfIterations parameter value.
*/
- public SVMLinearMultiClassClassificationTrainer withAmountOfIterations(int amountOfIterations) {
+ public SVMLinearMultiClassClassificationTrainer withAmountOfIterations(int amountOfIterations) {
this.amountOfIterations = amountOfIterations;
return this;
}
@@ -182,7 +240,7 @@ public class SVMLinearMultiClassClassificationTrainer
* @param amountOfLocIterations The parameter value.
* @return Trainer with new amountOfLocIterations parameter value.
*/
- public SVMLinearMultiClassClassificationTrainer withAmountOfLocIterations(int amountOfLocIterations) {
+ public SVMLinearMultiClassClassificationTrainer withAmountOfLocIterations(int amountOfLocIterations) {
this.amountOfLocIterations = amountOfLocIterations;
return this;
}
[39/50] [abbrv] ignite git commit: IGNITE-9093 Fixed
IgniteDbPutGetWithCacheStoreTest failing due to node shutdown without
checkpoint on stop - Fixes #4545.
Posted by ag...@apache.org.
IGNITE-9093 Fixed IgniteDbPutGetWithCacheStoreTest failing due to node shutdown without checkpoint on stop - Fixes #4545.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a549a742
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a549a742
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a549a742
Branch: refs/heads/ignite-5960
Commit: a549a742faf7c4708701dc2cbe54f3837c782db7
Parents: 19c990c
Author: Ilya Kasnacheev <il...@gmail.com>
Authored: Thu Sep 6 13:01:21 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Sep 6 13:01:21 2018 +0300
----------------------------------------------------------------------
.../processors/database/IgniteDbPutGetWithCacheStoreTest.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a549a742/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java
index 3238acf..8cca7f4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java
@@ -31,7 +31,6 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.IgniteReflectionFactory;
import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -124,7 +123,7 @@ public class IgniteDbPutGetWithCacheStoreTest extends GridCommonAbstractTest {
assertEquals(2000, storeMap.size());
- stopAllGrids();
+ stopAllGrids(false);
storeMap.clear();
@@ -158,7 +157,7 @@ public class IgniteDbPutGetWithCacheStoreTest extends GridCommonAbstractTest {
for (int i = 0; i < 2000; i++)
assertEquals(i, ig.cache(CACHE_NAME).get(i));
- stopAllGrids();
+ stopAllGrids(false);
storeMap.clear();
[10/50] [abbrv] ignite git commit: Revert "IGNITE-9448 Update
ZooKeeper version to 3.4.13 - Fixes #4661."
Posted by ag...@apache.org.
Revert "IGNITE-9448 Update ZooKeeper version to 3.4.13 - Fixes #4661."
This reverts commit 4d736fc20a6e667e841201ad2205f1440546191b.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d73e0350
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d73e0350
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d73e0350
Branch: refs/heads/ignite-5960
Commit: d73e035059bfeeda9dc680d0f729e16ccd2dc660
Parents: 254a652
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Sat Sep 1 23:17:27 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Sat Sep 1 23:17:27 2018 +0300
----------------------------------------------------------------------
parent/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d73e0350/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index f5b49ba..76e39b7 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -122,7 +122,7 @@
<yammer.metrics.core.version>2.2.0</yammer.metrics.core.version>
<yardstick.version>0.8.3</yardstick.version>
<zkclient.version>0.5</zkclient.version>
- <zookeeper.version>3.4.13</zookeeper.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
<!-- OSGI Manifest generation default property values -->
<osgi.import.package>*</osgi.import.package>
[38/50] [abbrv] ignite git commit: IGNITE-9422 Fixed NPE on clients
when new binary meta from joined node arrived. - Fixes #4681.
Posted by ag...@apache.org.
IGNITE-9422 Fixed NPE on clients when new binary meta from joined node arrived. - Fixes #4681.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/19c990ce
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/19c990ce
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/19c990ce
Branch: refs/heads/ignite-5960
Commit: 19c990ce58825f7c39deb4f524b7a60b328e286c
Parents: 7f22f08
Author: Ivan Daschinskiy <iv...@gmail.com>
Authored: Thu Sep 6 12:35:05 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Sep 6 12:54:28 2018 +0300
----------------------------------------------------------------------
.../binary/CacheObjectBinaryProcessorImpl.java | 9 +-
...cheObjectBinaryProcessorOnDiscoveryTest.java | 125 +++++++++++++++++++
.../ignite/testsuites/IgnitePdsTestSuite.java | 4 +
3 files changed, 135 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/19c990ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index d6b920a..4c101b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -492,7 +492,8 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
try {
BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta0);
- metadataFileStore.mergeAndWriteMetadata(mergedMeta);
+ if (!ctx.clientNode())
+ metadataFileStore.mergeAndWriteMetadata(mergedMeta);
metadataLocCache.put(typeId, new BinaryMetadataHolder(mergedMeta, 0, 0));
}
@@ -1016,7 +1017,8 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
localMetaHolder.pendingVersion(),
localMetaHolder.acceptedVersion()));
- metadataFileStore.writeMetadata(mergedMeta);
+ if (!ctx.clientNode())
+ metadataFileStore.writeMetadata(mergedMeta);
}
}
else {
@@ -1032,7 +1034,8 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
metadataLocCache.put(metaEntry.getKey(), newMetaHolder);
- metadataFileStore.writeMetadata(newMeta);
+ if (!ctx.clientNode())
+ metadataFileStore.writeMetadata(newMeta);
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/19c990ce/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest.java
new file mode 100644
index 0000000..279d3c8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.Arrays;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectBuilder;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.StopNodeFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ if ("client".equals(igniteInstanceName))
+ cfg.setClientMode(true).setFailureHandler(new StopNodeFailureHandler());
+
+ return cfg.setDiscoverySpi(new TcpDiscoverySpi()
+ .setIpFinder(IP_FINDER))
+ .setDataStorageConfiguration(new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setPersistenceEnabled(true)));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * Tests that joining node metadata correctly handled on client.
+ * @throws Exception If fails.
+ */
+ public void testJoiningNodeBinaryMetaOnClient() throws Exception {
+ IgniteEx ig0 = (IgniteEx)startGrids(2);
+
+ ig0.cluster().active(true);
+
+ addBinaryType(ig0, "test_1", new IgniteBiTuple<>("name", String.class));
+
+ stopGrid(0);
+
+ Ignite ig1 = grid(1);
+
+ // Modify existing type.
+ addBinaryType(ig1, "test_1", new IgniteBiTuple<>("id", Integer.class));
+
+ // Add new type.
+ addBinaryType(ig1, "test_2", new IgniteBiTuple<>("name", String.class));
+
+ stopGrid(1);
+
+ startGrid(0);
+
+ IgniteEx client = startGrid(getConfiguration("client"));
+
+ startGrid(1);
+
+ awaitPartitionMapExchange();
+
+ // Check that new metadata from grid_1 was handled without NPE on client.
+ assertNull(client.context().failure().failureContext());
+
+ // Check that metadata from grid_1 correctly loaded on client.
+ assertTrue(client.binary().type("test_1").fieldNames().containsAll(Arrays.asList("id", "name")));
+ assertTrue(client.binary().type("test_2").fieldNames().contains("name"));
+ }
+
+ /**
+ * @param ig Ig.
+ * @param typeName Type name.
+ * @param fields Fields.
+ */
+ @SafeVarargs
+ private final BinaryObject addBinaryType(Ignite ig, String typeName, IgniteBiTuple<String, Class<?>>... fields) {
+ BinaryObjectBuilder builder = ig.binary().builder(typeName);
+
+ if (fields != null) {
+ for (IgniteBiTuple<String,Class<?>> field: fields)
+ builder.setField(field.get1(), field.get2());
+ }
+
+ return builder.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/19c990ce/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index 0a995da..9f50b32 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -20,6 +20,7 @@ package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.internal.pagemem.impl.PageMemoryNoLoadSelfTest;
import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistence;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheWithoutCheckpointsTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheConfigurationFileConsistencyCheckTest;
@@ -95,6 +96,9 @@ public class IgnitePdsTestSuite extends TestSuite {
suite.addTestSuite(ExplicitWalDeltaConsistencyTest.class);
suite.addTestSuite(SysPropWalDeltaConsistencyTest.class);
+ // Binary meta tests.
+ suite.addTestSuite(IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest.class);
+
return suite;
}
[44/50] [abbrv] ignite git commit: IGNITE-9483: MVCC compatibility
fixes. This closes #4696.
Posted by ag...@apache.org.
IGNITE-9483: MVCC compatibility fixes. This closes #4696.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/469fb49e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/469fb49e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/469fb49e
Branch: refs/heads/ignite-5960
Commit: 469fb49e43e88bf7e007da9ba8853ec953a82aa1
Parents: a232b88
Author: Igor Seliverstov <gv...@gmail.com>
Authored: Fri Sep 7 11:57:46 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Sep 7 11:57:46 2018 +0300
----------------------------------------------------------------------
.../ignite/codegen/MessageCodeGenerator.java | 4 +-
.../internal/jdbc/thin/JdbcThinTcpIo.java | 25 ++++++-----
.../managers/communication/GridIoManager.java | 2 +-
.../distributed/dht/GridDhtTxFinishRequest.java | 24 +++++------
.../dht/GridDhtTxPrepareRequest.java | 44 ++++++++++----------
.../GridDhtPartitionsSingleMessage.java | 40 +++++++++---------
.../distributed/near/GridNearGetRequest.java | 24 +++++------
.../near/GridNearSingleGetRequest.java | 16 +++----
.../near/GridNearTxPrepareResponse.java | 24 +++++------
.../cache/query/GridCacheQueryRequest.java | 44 ++++++++++----------
.../odbc/jdbc/JdbcBatchExecuteRequest.java | 40 +++++++++---------
.../odbc/jdbc/JdbcBatchExecuteResult.java | 11 +++--
.../odbc/jdbc/JdbcBulkLoadAckResult.java | 11 +++--
.../odbc/jdbc/JdbcBulkLoadBatchRequest.java | 11 +++--
.../processors/odbc/jdbc/JdbcColumnMeta.java | 7 +++-
.../processors/odbc/jdbc/JdbcColumnMetaV2.java | 11 +++--
.../processors/odbc/jdbc/JdbcColumnMetaV3.java | 11 +++--
.../processors/odbc/jdbc/JdbcColumnMetaV4.java | 11 +++--
.../odbc/jdbc/JdbcConnectionContext.java | 6 ++-
.../processors/odbc/jdbc/JdbcIndexMeta.java | 7 +++-
.../processors/odbc/jdbc/JdbcMessageParser.java | 13 ++++--
.../odbc/jdbc/JdbcMetaColumnsRequest.java | 11 +++--
.../odbc/jdbc/JdbcMetaColumnsResult.java | 15 ++++---
.../odbc/jdbc/JdbcMetaIndexesRequest.java | 11 +++--
.../odbc/jdbc/JdbcMetaIndexesResult.java | 15 ++++---
.../odbc/jdbc/JdbcMetaParamsRequest.java | 11 +++--
.../odbc/jdbc/JdbcMetaParamsResult.java | 15 ++++---
.../odbc/jdbc/JdbcMetaPrimaryKeysRequest.java | 11 +++--
.../odbc/jdbc/JdbcMetaPrimaryKeysResult.java | 15 ++++---
.../odbc/jdbc/JdbcMetaSchemasRequest.java | 11 +++--
.../odbc/jdbc/JdbcMetaSchemasResult.java | 11 +++--
.../odbc/jdbc/JdbcMetaTablesRequest.java | 11 +++--
.../odbc/jdbc/JdbcMetaTablesResult.java | 15 ++++---
.../jdbc/JdbcOrderedBatchExecuteRequest.java | 9 ++--
.../jdbc/JdbcOrderedBatchExecuteResult.java | 11 +++--
.../processors/odbc/jdbc/JdbcParameterMeta.java | 7 +++-
.../odbc/jdbc/JdbcPrimaryKeyMeta.java | 7 +++-
.../processors/odbc/jdbc/JdbcQuery.java | 7 +++-
.../odbc/jdbc/JdbcQueryCloseRequest.java | 11 +++--
...dbcQueryExecuteMultipleStatementsResult.java | 15 ++++---
.../odbc/jdbc/JdbcQueryExecuteRequest.java | 33 ++++++++-------
.../odbc/jdbc/JdbcQueryExecuteResult.java | 11 +++--
.../odbc/jdbc/JdbcQueryFetchRequest.java | 11 +++--
.../odbc/jdbc/JdbcQueryFetchResult.java | 11 +++--
.../odbc/jdbc/JdbcQueryMetadataRequest.java | 11 +++--
.../odbc/jdbc/JdbcQueryMetadataResult.java | 15 ++++---
.../odbc/jdbc/JdbcRawBinarylizable.java | 8 +++-
.../processors/odbc/jdbc/JdbcRequest.java | 13 ++++--
.../processors/odbc/jdbc/JdbcResponse.java | 11 +++--
.../processors/odbc/jdbc/JdbcResult.java | 12 ++++--
.../processors/odbc/jdbc/JdbcResultInfo.java | 7 +++-
.../processors/odbc/jdbc/JdbcTableMeta.java | 7 +++-
.../odbc/odbc/OdbcConnectionContext.java | 2 +-
.../processors/odbc/odbc/OdbcMessageParser.java | 4 +-
.../h2/twostep/msg/GridH2QueryRequest.java | 44 ++++++++++----------
55 files changed, 466 insertions(+), 339 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
----------------------------------------------------------------------
diff --git a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
index 51365f3..9512bae 100644
--- a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
+++ b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
@@ -38,8 +38,6 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxQueryEnlistRequest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxQueryResultsEnlistRequest;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.GridCodegenConverter;
import org.apache.ignite.internal.GridDirectCollection;
@@ -170,7 +168,7 @@ public class MessageCodeGenerator {
// gen.generateAll(true);
- gen.generateAndWrite(GridNearTxQueryResultsEnlistRequest.class);
+// gen.generateAndWrite(GridNearTxQueryResultsEnlistRequest.class);
// gen.generateAndWrite(GridNearAtomicUpdateRequest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
index 2c3f321..6155423 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
@@ -74,7 +74,7 @@ public class JdbcThinTcpIo {
private static final ClientListenerProtocolVersion VER_2_7_0 = ClientListenerProtocolVersion.create(2, 7, 0);
/** Current version. */
- private static final ClientListenerProtocolVersion CURRENT_VER = VER_2_7_0;
+ public static final ClientListenerProtocolVersion CURRENT_VER = VER_2_7_0;
/** Initial output stream capacity for handshake. */
private static final int HANDSHAKE_MSG_SIZE = 13;
@@ -326,7 +326,9 @@ public class JdbcThinTcpIo {
writer.writeBoolean(connProps.isAutoCloseServerCursor());
writer.writeBoolean(connProps.isLazy());
writer.writeBoolean(connProps.isSkipReducerOnUpdate());
- writer.writeString(connProps.nestedTxMode());
+
+ if (ver.compareTo(VER_2_7_0) >= 0)
+ writer.writeString(connProps.nestedTxMode());
if (!F.isEmpty(connProps.getUsername())) {
assert ver.compareTo(VER_2_5_0) >= 0 : "Authentication is supported since 2.5";
@@ -375,15 +377,16 @@ public class JdbcThinTcpIo {
+ ", url=" + connProps.getUrl() + ']', SqlStateCode.CONNECTION_REJECTED);
}
- if (VER_2_4_0.equals(srvProtocolVer)
- || VER_2_3_0.equals(srvProtocolVer)
- || VER_2_1_5.equals(srvProtocolVer))
- handshake(srvProtocolVer);
- else if (VER_2_1_0.equals(srvProtocolVer))
+ if (VER_2_5_0.equals(srvProtoVer0)
+ || VER_2_4_0.equals(srvProtoVer0)
+ || VER_2_3_0.equals(srvProtoVer0)
+ || VER_2_1_5.equals(srvProtoVer0))
+ handshake(srvProtoVer0);
+ else if (VER_2_1_0.equals(srvProtoVer0))
handshake_2_1_0();
else {
throw new SQLException("Handshake failed [driverProtocolVer=" + CURRENT_VER +
- ", remoteNodeProtocolVer=" + srvProtocolVer + ", err=" + err + ']',
+ ", remoteNodeProtocolVer=" + srvProtoVer0 + ", err=" + err + ']',
SqlStateCode.CONNECTION_REJECTED);
}
}
@@ -466,7 +469,7 @@ public class JdbcThinTcpIo {
BinaryWriterExImpl writer = new BinaryWriterExImpl(null, new BinaryHeapOutputStream(cap),
null, null);
- req.writeBinary(writer);
+ req.writeBinary(writer, srvProtocolVer);
send(writer.array());
}
@@ -500,7 +503,7 @@ public class JdbcThinTcpIo {
BinaryWriterExImpl writer = new BinaryWriterExImpl(null, new BinaryHeapOutputStream(cap), null, null);
- req.writeBinary(writer);
+ req.writeBinary(writer, srvProtocolVer);
send(writer.array());
@@ -523,7 +526,7 @@ public class JdbcThinTcpIo {
JdbcResponse res = new JdbcResponse();
- res.readBinary(reader);
+ res.readBinary(reader, srvProtocolVer);
return res;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index f515d57..b3c80b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -1662,7 +1662,7 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
throw (ClusterTopologyCheckedException)e.getCause();
if (!ctx.discovery().alive(node))
- throw new ClusterTopologyCheckedException("Failed to send message, node left: " + node.id());
+ throw new ClusterTopologyCheckedException("Failed to send message, node left: " + node.id(), e);
throw new IgniteCheckedException("Failed to send message (node may have left the grid or " +
"TCP connection cannot be established due to firewall issues) " +
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
index 2696c0e..b1908ff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
@@ -397,37 +397,37 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
writer.incrementState();
case 23:
- if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
+ if (!writer.writeUuid("nearNodeId", nearNodeId))
return false;
writer.incrementState();
case 24:
- if (!writer.writeUuid("nearNodeId", nearNodeId))
+ if (!writer.writeMessage("partUpdateCnt", partUpdateCnt))
return false;
writer.incrementState();
case 25:
- if (!writer.writeMessage("partUpdateCnt", partUpdateCnt))
+ if (!writer.writeCollection("pendingVers", pendingVers, MessageCollectionItemType.MSG))
return false;
writer.incrementState();
case 26:
- if (!writer.writeCollection("pendingVers", pendingVers, MessageCollectionItemType.MSG))
+ if (!writer.writeMessage("writeVer", writeVer))
return false;
writer.incrementState();
case 27:
- if (!writer.writeMap("updCntrs", updCntrs, MessageCollectionItemType.INT, MessageCollectionItemType.MSG))
+ if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
return false;
writer.incrementState();
case 28:
- if (!writer.writeMessage("writeVer", writeVer))
+ if (!writer.writeMap("updCntrs", updCntrs, MessageCollectionItemType.INT, MessageCollectionItemType.MSG))
return false;
writer.incrementState();
@@ -469,7 +469,7 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
reader.incrementState();
case 23:
- mvccSnapshot = reader.readMessage("mvccSnapshot");
+ nearNodeId = reader.readUuid("nearNodeId");
if (!reader.isLastRead())
return false;
@@ -477,7 +477,7 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
reader.incrementState();
case 24:
- nearNodeId = reader.readUuid("nearNodeId");
+ partUpdateCnt = reader.readMessage("partUpdateCnt");
if (!reader.isLastRead())
return false;
@@ -485,7 +485,7 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
reader.incrementState();
case 25:
- partUpdateCnt = reader.readMessage("partUpdateCnt");
+ pendingVers = reader.readCollection("pendingVers", MessageCollectionItemType.MSG);
if (!reader.isLastRead())
return false;
@@ -493,7 +493,7 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
reader.incrementState();
case 26:
- pendingVers = reader.readCollection("pendingVers", MessageCollectionItemType.MSG);
+ writeVer = reader.readMessage("writeVer");
if (!reader.isLastRead())
return false;
@@ -501,7 +501,7 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
reader.incrementState();
case 27:
- updCntrs = reader.readMap("updCntrs", MessageCollectionItemType.INT, MessageCollectionItemType.MSG, false);
+ mvccSnapshot = reader.readMessage("mvccSnapshot");
if (!reader.isLastRead())
return false;
@@ -509,7 +509,7 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
reader.incrementState();
case 28:
- writeVer = reader.readMessage("writeVer");
+ updCntrs = reader.readMap("updCntrs", MessageCollectionItemType.INT, MessageCollectionItemType.MSG, false);
if (!reader.isLastRead())
return false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
index a71fb6c..68c1f39 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
@@ -427,67 +427,67 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
writer.incrementState();
case 23:
- if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
+ if (!writer.writeUuid("nearNodeId", nearNodeId))
return false;
writer.incrementState();
case 24:
- if (!writer.writeUuid("nearNodeId", nearNodeId))
+ if (!writer.writeCollection("nearWrites", nearWrites, MessageCollectionItemType.MSG))
return false;
writer.incrementState();
case 25:
- if (!writer.writeCollection("nearWrites", nearWrites, MessageCollectionItemType.MSG))
+ if (!writer.writeMessage("nearXidVer", nearXidVer))
return false;
writer.incrementState();
case 26:
- if (!writer.writeMessage("nearXidVer", nearXidVer))
+ if (!writer.writeCollection("ownedKeys", ownedKeys, MessageCollectionItemType.MSG))
return false;
writer.incrementState();
case 27:
- if (!writer.writeCollection("ownedKeys", ownedKeys, MessageCollectionItemType.MSG))
+ if (!writer.writeCollection("ownedVals", ownedVals, MessageCollectionItemType.MSG))
return false;
writer.incrementState();
case 28:
- if (!writer.writeCollection("ownedVals", ownedVals, MessageCollectionItemType.MSG))
+ if (!writer.writeBitSet("preloadKeys", preloadKeys))
return false;
writer.incrementState();
case 29:
- if (!writer.writeBitSet("preloadKeys", preloadKeys))
+ if (!writer.writeBoolean("skipCompletedVers", skipCompletedVers))
return false;
writer.incrementState();
case 30:
- if (!writer.writeBoolean("skipCompletedVers", skipCompletedVers))
+ if (!writer.writeUuid("subjId", subjId))
return false;
writer.incrementState();
case 31:
- if (!writer.writeUuid("subjId", subjId))
+ if (!writer.writeInt("taskNameHash", taskNameHash))
return false;
writer.incrementState();
case 32:
- if (!writer.writeInt("taskNameHash", taskNameHash))
+ if (!writer.writeMessage("topVer", topVer))
return false;
writer.incrementState();
case 33:
- if (!writer.writeMessage("topVer", topVer))
+ if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
return false;
writer.incrementState();
@@ -533,7 +533,7 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
reader.incrementState();
case 23:
- mvccSnapshot = reader.readMessage("mvccSnapshot");
+ nearNodeId = reader.readUuid("nearNodeId");
if (!reader.isLastRead())
return false;
@@ -541,7 +541,7 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
reader.incrementState();
case 24:
- nearNodeId = reader.readUuid("nearNodeId");
+ nearWrites = reader.readCollection("nearWrites", MessageCollectionItemType.MSG);
if (!reader.isLastRead())
return false;
@@ -549,7 +549,7 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
reader.incrementState();
case 25:
- nearWrites = reader.readCollection("nearWrites", MessageCollectionItemType.MSG);
+ nearXidVer = reader.readMessage("nearXidVer");
if (!reader.isLastRead())
return false;
@@ -557,7 +557,7 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
reader.incrementState();
case 26:
- nearXidVer = reader.readMessage("nearXidVer");
+ ownedKeys = reader.readCollection("ownedKeys", MessageCollectionItemType.MSG);
if (!reader.isLastRead())
return false;
@@ -565,7 +565,7 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
reader.incrementState();
case 27:
- ownedKeys = reader.readCollection("ownedKeys", MessageCollectionItemType.MSG);
+ ownedVals = reader.readCollection("ownedVals", MessageCollectionItemType.MSG);
if (!reader.isLastRead())
return false;
@@ -573,7 +573,7 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
reader.incrementState();
case 28:
- ownedVals = reader.readCollection("ownedVals", MessageCollectionItemType.MSG);
+ preloadKeys = reader.readBitSet("preloadKeys");
if (!reader.isLastRead())
return false;
@@ -581,7 +581,7 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
reader.incrementState();
case 29:
- preloadKeys = reader.readBitSet("preloadKeys");
+ skipCompletedVers = reader.readBoolean("skipCompletedVers");
if (!reader.isLastRead())
return false;
@@ -589,7 +589,7 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
reader.incrementState();
case 30:
- skipCompletedVers = reader.readBoolean("skipCompletedVers");
+ subjId = reader.readUuid("subjId");
if (!reader.isLastRead())
return false;
@@ -597,7 +597,7 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
reader.incrementState();
case 31:
- subjId = reader.readUuid("subjId");
+ taskNameHash = reader.readInt("taskNameHash");
if (!reader.isLastRead())
return false;
@@ -605,7 +605,7 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
reader.incrementState();
case 32:
- taskNameHash = reader.readInt("taskNameHash");
+ topVer = reader.readMessage("topVer");
if (!reader.isLastRead())
return false;
@@ -613,7 +613,7 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
reader.incrementState();
case 33:
- topVer = reader.readMessage("topVer");
+ mvccSnapshot = reader.readMessage("mvccSnapshot");
if (!reader.isLastRead())
return false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index 77d7eab..7998e07 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -474,61 +474,61 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
switch (writer.state()) {
case 5:
- if (!writer.writeMessage("activeQryTrackers", activeQryTrackers))
+ if (!writer.writeBoolean("client", client))
return false;
writer.incrementState();
case 6:
- if (!writer.writeBoolean("client", client))
+ if (!writer.writeMap("dupPartsData", dupPartsData, MessageCollectionItemType.INT, MessageCollectionItemType.INT))
return false;
writer.incrementState();
case 7:
- if (!writer.writeMap("dupPartsData", dupPartsData, MessageCollectionItemType.INT, MessageCollectionItemType.INT))
+ if (!writer.writeByteArray("errBytes", errBytes))
return false;
writer.incrementState();
case 8:
- if (!writer.writeByteArray("errBytes", errBytes))
+ if (!writer.writeMessage("finishMsg", finishMsg))
return false;
writer.incrementState();
case 9:
- if (!writer.writeMessage("finishMsg", finishMsg))
+ if (!writer.writeCollection("grpsAffRequest", grpsAffRequest, MessageCollectionItemType.INT))
return false;
writer.incrementState();
case 10:
- if (!writer.writeCollection("grpsAffRequest", grpsAffRequest, MessageCollectionItemType.INT))
+ if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes))
return false;
writer.incrementState();
case 11:
- if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes))
+ if (!writer.writeByteArray("partHistCntrsBytes", partHistCntrsBytes))
return false;
writer.incrementState();
case 12:
- if (!writer.writeByteArray("partHistCntrsBytes", partHistCntrsBytes))
+ if (!writer.writeByteArray("partsBytes", partsBytes))
return false;
writer.incrementState();
case 13:
- if (!writer.writeByteArray("partsBytes", partsBytes))
+ if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes))
return false;
writer.incrementState();
case 14:
- if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes))
+ if (!writer.writeMessage("activeQryTrackers", activeQryTrackers))
return false;
writer.incrementState();
@@ -549,7 +549,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
switch (reader.state()) {
case 5:
- activeQryTrackers = reader.readMessage("activeQryTrackers");
+ client = reader.readBoolean("client");
if (!reader.isLastRead())
return false;
@@ -557,7 +557,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
reader.incrementState();
case 6:
- client = reader.readBoolean("client");
+ dupPartsData = reader.readMap("dupPartsData", MessageCollectionItemType.INT, MessageCollectionItemType.INT, false);
if (!reader.isLastRead())
return false;
@@ -565,7 +565,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
reader.incrementState();
case 7:
- dupPartsData = reader.readMap("dupPartsData", MessageCollectionItemType.INT, MessageCollectionItemType.INT, false);
+ errBytes = reader.readByteArray("errBytes");
if (!reader.isLastRead())
return false;
@@ -573,7 +573,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
reader.incrementState();
case 8:
- errBytes = reader.readByteArray("errBytes");
+ finishMsg = reader.readMessage("finishMsg");
if (!reader.isLastRead())
return false;
@@ -581,7 +581,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
reader.incrementState();
case 9:
- finishMsg = reader.readMessage("finishMsg");
+ grpsAffRequest = reader.readCollection("grpsAffRequest", MessageCollectionItemType.INT);
if (!reader.isLastRead())
return false;
@@ -589,7 +589,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
reader.incrementState();
case 10:
- grpsAffRequest = reader.readCollection("grpsAffRequest", MessageCollectionItemType.INT);
+ partCntrsBytes = reader.readByteArray("partCntrsBytes");
if (!reader.isLastRead())
return false;
@@ -597,7 +597,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
reader.incrementState();
case 11:
- partCntrsBytes = reader.readByteArray("partCntrsBytes");
+ partHistCntrsBytes = reader.readByteArray("partHistCntrsBytes");
if (!reader.isLastRead())
return false;
@@ -605,7 +605,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
reader.incrementState();
case 12:
- partHistCntrsBytes = reader.readByteArray("partHistCntrsBytes");
+ partsBytes = reader.readByteArray("partsBytes");
if (!reader.isLastRead())
return false;
@@ -613,7 +613,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
reader.incrementState();
case 13:
- partsBytes = reader.readByteArray("partsBytes");
+ partsSizesBytes = reader.readByteArray("partsSizesBytes");
if (!reader.isLastRead())
return false;
@@ -621,7 +621,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
reader.incrementState();
case 14:
- partsSizesBytes = reader.readByteArray("partsSizesBytes");
+ activeQryTrackers = reader.readMessage("activeQryTrackers");
if (!reader.isLastRead())
return false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
index c891bfb..f594e2b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
@@ -397,37 +397,37 @@ public class GridNearGetRequest extends GridCacheIdMessage implements GridCacheD
writer.incrementState();
case 9:
- if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
+ if (!writer.writeCollection("readersFlags", readersFlags, MessageCollectionItemType.BOOLEAN))
return false;
writer.incrementState();
case 10:
- if (!writer.writeCollection("readersFlags", readersFlags, MessageCollectionItemType.BOOLEAN))
+ if (!writer.writeUuid("subjId", subjId))
return false;
writer.incrementState();
case 11:
- if (!writer.writeUuid("subjId", subjId))
+ if (!writer.writeInt("taskNameHash", taskNameHash))
return false;
writer.incrementState();
case 12:
- if (!writer.writeInt("taskNameHash", taskNameHash))
+ if (!writer.writeMessage("topVer", topVer))
return false;
writer.incrementState();
case 13:
- if (!writer.writeMessage("topVer", topVer))
+ if (!writer.writeMessage("ver", ver))
return false;
writer.incrementState();
case 14:
- if (!writer.writeMessage("ver", ver))
+ if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
return false;
writer.incrementState();
@@ -497,7 +497,7 @@ public class GridNearGetRequest extends GridCacheIdMessage implements GridCacheD
reader.incrementState();
case 9:
- mvccSnapshot = reader.readMessage("mvccSnapshot");
+ readersFlags = reader.readCollection("readersFlags", MessageCollectionItemType.BOOLEAN);
if (!reader.isLastRead())
return false;
@@ -505,7 +505,7 @@ public class GridNearGetRequest extends GridCacheIdMessage implements GridCacheD
reader.incrementState();
case 10:
- readersFlags = reader.readCollection("readersFlags", MessageCollectionItemType.BOOLEAN);
+ subjId = reader.readUuid("subjId");
if (!reader.isLastRead())
return false;
@@ -513,7 +513,7 @@ public class GridNearGetRequest extends GridCacheIdMessage implements GridCacheD
reader.incrementState();
case 11:
- subjId = reader.readUuid("subjId");
+ taskNameHash = reader.readInt("taskNameHash");
if (!reader.isLastRead())
return false;
@@ -521,7 +521,7 @@ public class GridNearGetRequest extends GridCacheIdMessage implements GridCacheD
reader.incrementState();
case 12:
- taskNameHash = reader.readInt("taskNameHash");
+ topVer = reader.readMessage("topVer");
if (!reader.isLastRead())
return false;
@@ -529,7 +529,7 @@ public class GridNearGetRequest extends GridCacheIdMessage implements GridCacheD
reader.incrementState();
case 13:
- topVer = reader.readMessage("topVer");
+ ver = reader.readMessage("ver");
if (!reader.isLastRead())
return false;
@@ -537,7 +537,7 @@ public class GridNearGetRequest extends GridCacheIdMessage implements GridCacheD
reader.incrementState();
case 14:
- ver = reader.readMessage("ver");
+ mvccSnapshot = reader.readMessage("mvccSnapshot");
if (!reader.isLastRead())
return false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
index d1aa2e9..cf885e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
@@ -337,7 +337,7 @@ public class GridNearSingleGetRequest extends GridCacheIdMessage implements Grid
reader.incrementState();
case 8:
- mvccSnapshot = reader.readMessage("mvccSnapshot");
+ subjId = reader.readUuid("subjId");
if (!reader.isLastRead())
return false;
@@ -345,7 +345,7 @@ public class GridNearSingleGetRequest extends GridCacheIdMessage implements Grid
reader.incrementState();
case 9:
- subjId = reader.readUuid("subjId");
+ taskNameHash = reader.readInt("taskNameHash");
if (!reader.isLastRead())
return false;
@@ -353,7 +353,7 @@ public class GridNearSingleGetRequest extends GridCacheIdMessage implements Grid
reader.incrementState();
case 10:
- taskNameHash = reader.readInt("taskNameHash");
+ topVer = reader.readMessage("topVer");
if (!reader.isLastRead())
return false;
@@ -361,7 +361,7 @@ public class GridNearSingleGetRequest extends GridCacheIdMessage implements Grid
reader.incrementState();
case 11:
- topVer = reader.readMessage("topVer");
+ mvccSnapshot = reader.readMessage("mvccSnapshot");
if (!reader.isLastRead())
return false;
@@ -419,25 +419,25 @@ public class GridNearSingleGetRequest extends GridCacheIdMessage implements Grid
writer.incrementState();
case 8:
- if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
+ if (!writer.writeUuid("subjId", subjId))
return false;
writer.incrementState();
case 9:
- if (!writer.writeUuid("subjId", subjId))
+ if (!writer.writeInt("taskNameHash", taskNameHash))
return false;
writer.incrementState();
case 10:
- if (!writer.writeInt("taskNameHash", taskNameHash))
+ if (!writer.writeMessage("topVer", topVer))
return false;
writer.incrementState();
case 11:
- if (!writer.writeMessage("topVer", topVer))
+ if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
return false;
writer.incrementState();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
index 92db463..e9865df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
@@ -407,37 +407,37 @@ public class GridNearTxPrepareResponse extends GridDistributedTxPrepareResponse
writer.incrementState();
case 15:
- if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
+ if (!writer.writeCollection("ownedValKeys", ownedValKeys, MessageCollectionItemType.MSG))
return false;
writer.incrementState();
case 16:
- if (!writer.writeCollection("ownedValKeys", ownedValKeys, MessageCollectionItemType.MSG))
+ if (!writer.writeCollection("ownedValVals", ownedValVals, MessageCollectionItemType.MSG))
return false;
writer.incrementState();
case 17:
- if (!writer.writeCollection("ownedValVals", ownedValVals, MessageCollectionItemType.MSG))
+ if (!writer.writeCollection("pending", pending, MessageCollectionItemType.MSG))
return false;
writer.incrementState();
case 18:
- if (!writer.writeCollection("pending", pending, MessageCollectionItemType.MSG))
+ if (!writer.writeMessage("retVal", retVal))
return false;
writer.incrementState();
case 19:
- if (!writer.writeMessage("retVal", retVal))
+ if (!writer.writeMessage("writeVer", writeVer))
return false;
writer.incrementState();
case 20:
- if (!writer.writeMessage("writeVer", writeVer))
+ if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
return false;
writer.incrementState();
@@ -499,7 +499,7 @@ public class GridNearTxPrepareResponse extends GridDistributedTxPrepareResponse
reader.incrementState();
case 15:
- mvccSnapshot = reader.readMessage("mvccSnapshot");
+ ownedValKeys = reader.readCollection("ownedValKeys", MessageCollectionItemType.MSG);
if (!reader.isLastRead())
return false;
@@ -507,7 +507,7 @@ public class GridNearTxPrepareResponse extends GridDistributedTxPrepareResponse
reader.incrementState();
case 16:
- ownedValKeys = reader.readCollection("ownedValKeys", MessageCollectionItemType.MSG);
+ ownedValVals = reader.readCollection("ownedValVals", MessageCollectionItemType.MSG);
if (!reader.isLastRead())
return false;
@@ -515,7 +515,7 @@ public class GridNearTxPrepareResponse extends GridDistributedTxPrepareResponse
reader.incrementState();
case 17:
- ownedValVals = reader.readCollection("ownedValVals", MessageCollectionItemType.MSG);
+ pending = reader.readCollection("pending", MessageCollectionItemType.MSG);
if (!reader.isLastRead())
return false;
@@ -523,7 +523,7 @@ public class GridNearTxPrepareResponse extends GridDistributedTxPrepareResponse
reader.incrementState();
case 18:
- pending = reader.readCollection("pending", MessageCollectionItemType.MSG);
+ retVal = reader.readMessage("retVal");
if (!reader.isLastRead())
return false;
@@ -531,7 +531,7 @@ public class GridNearTxPrepareResponse extends GridDistributedTxPrepareResponse
reader.incrementState();
case 19:
- retVal = reader.readMessage("retVal");
+ writeVer = reader.readMessage("writeVer");
if (!reader.isLastRead())
return false;
@@ -539,7 +539,7 @@ public class GridNearTxPrepareResponse extends GridDistributedTxPrepareResponse
reader.incrementState();
case 20:
- writeVer = reader.readMessage("writeVer");
+ mvccSnapshot = reader.readMessage("mvccSnapshot");
if (!reader.isLastRead())
return false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
index ebbca35..b7205b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
@@ -586,55 +586,55 @@ public class GridCacheQueryRequest extends GridCacheIdMessage implements GridCac
writer.incrementState();
case 15:
- if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
+ if (!writer.writeInt("pageSize", pageSize))
return false;
writer.incrementState();
case 16:
- if (!writer.writeInt("pageSize", pageSize))
+ if (!writer.writeInt("part", part))
return false;
writer.incrementState();
case 17:
- if (!writer.writeInt("part", part))
+ if (!writer.writeByteArray("rdcBytes", rdcBytes))
return false;
writer.incrementState();
case 18:
- if (!writer.writeByteArray("rdcBytes", rdcBytes))
+ if (!writer.writeUuid("subjId", subjId))
return false;
writer.incrementState();
case 19:
- if (!writer.writeUuid("subjId", subjId))
+ if (!writer.writeInt("taskHash", taskHash))
return false;
writer.incrementState();
case 20:
- if (!writer.writeInt("taskHash", taskHash))
+ if (!writer.writeMessage("topVer", topVer))
return false;
writer.incrementState();
case 21:
- if (!writer.writeMessage("topVer", topVer))
+ if (!writer.writeByteArray("transBytes", transBytes))
return false;
writer.incrementState();
case 22:
- if (!writer.writeByteArray("transBytes", transBytes))
+ if (!writer.writeByte("type", type != null ? (byte)type.ordinal() : -1))
return false;
writer.incrementState();
case 23:
- if (!writer.writeByte("type", type != null ? (byte)type.ordinal() : -1))
+ if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
return false;
writer.incrementState();
@@ -752,7 +752,7 @@ public class GridCacheQueryRequest extends GridCacheIdMessage implements GridCac
reader.incrementState();
case 15:
- mvccSnapshot = reader.readMessage("mvccSnapshot");
+ pageSize = reader.readInt("pageSize");
if (!reader.isLastRead())
return false;
@@ -760,7 +760,7 @@ public class GridCacheQueryRequest extends GridCacheIdMessage implements GridCac
reader.incrementState();
case 16:
- pageSize = reader.readInt("pageSize");
+ part = reader.readInt("part");
if (!reader.isLastRead())
return false;
@@ -768,7 +768,7 @@ public class GridCacheQueryRequest extends GridCacheIdMessage implements GridCac
reader.incrementState();
case 17:
- part = reader.readInt("part");
+ rdcBytes = reader.readByteArray("rdcBytes");
if (!reader.isLastRead())
return false;
@@ -776,7 +776,7 @@ public class GridCacheQueryRequest extends GridCacheIdMessage implements GridCac
reader.incrementState();
case 18:
- rdcBytes = reader.readByteArray("rdcBytes");
+ subjId = reader.readUuid("subjId");
if (!reader.isLastRead())
return false;
@@ -784,7 +784,7 @@ public class GridCacheQueryRequest extends GridCacheIdMessage implements GridCac
reader.incrementState();
case 19:
- subjId = reader.readUuid("subjId");
+ taskHash = reader.readInt("taskHash");
if (!reader.isLastRead())
return false;
@@ -792,7 +792,7 @@ public class GridCacheQueryRequest extends GridCacheIdMessage implements GridCac
reader.incrementState();
case 20:
- taskHash = reader.readInt("taskHash");
+ topVer = reader.readMessage("topVer");
if (!reader.isLastRead())
return false;
@@ -800,7 +800,7 @@ public class GridCacheQueryRequest extends GridCacheIdMessage implements GridCac
reader.incrementState();
case 21:
- topVer = reader.readMessage("topVer");
+ transBytes = reader.readByteArray("transBytes");
if (!reader.isLastRead())
return false;
@@ -808,23 +808,23 @@ public class GridCacheQueryRequest extends GridCacheIdMessage implements GridCac
reader.incrementState();
case 22:
- transBytes = reader.readByteArray("transBytes");
+ byte typeOrd;
+
+ typeOrd = reader.readByte("type");
if (!reader.isLastRead())
return false;
+ type = GridCacheQueryType.fromOrdinal(typeOrd);
+
reader.incrementState();
case 23:
- byte typeOrd;
-
- typeOrd = reader.readByte("type");
+ mvccSnapshot = reader.readMessage("mvccSnapshot");
if (!reader.isLastRead())
return false;
- type = GridCacheQueryType.fromOrdinal(typeOrd);
-
reader.incrementState();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java
index 065a298..404a1c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java
@@ -17,17 +17,20 @@
package org.apache.ignite.internal.processors.odbc.jdbc;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.internal.processors.odbc.jdbc.JdbcConnectionContext.VER_2_4_0;
+import static org.apache.ignite.internal.processors.odbc.jdbc.JdbcConnectionContext.VER_2_7_0;
+
/**
* JDBC batch execute request.
*/
@@ -129,8 +132,8 @@ public class JdbcBatchExecuteRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer, ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeString(schemaName);
@@ -138,20 +141,23 @@ public class JdbcBatchExecuteRequest extends JdbcRequest {
writer.writeInt(queries.size());
for (JdbcQuery q : queries)
- q.writeBinary(writer);
+ q.writeBinary(writer, ver);
}
else
writer.writeInt(0);
- writer.writeBoolean(autoCommit);
- writer.writeBoolean(lastStreamBatch);
+ if (ver.compareTo(VER_2_4_0) >= 0)
+ writer.writeBoolean(lastStreamBatch);
+
+ if (ver.compareTo(VER_2_7_0) >= 0)
+ writer.writeBoolean(autoCommit);
}
/** {@inheritDoc} */
@SuppressWarnings("SimplifiableIfStatement")
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader, ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
schemaName = reader.readString();
@@ -162,22 +168,16 @@ public class JdbcBatchExecuteRequest extends JdbcRequest {
for (int i = 0; i < n; ++i) {
JdbcQuery qry = new JdbcQuery();
- qry.readBinary(reader);
+ qry.readBinary(reader, ver);
queries.add(qry);
}
- try {
- if (reader.available() > 0) {
- autoCommit = reader.readBoolean();
- lastStreamBatch = reader.readBoolean();
- }
- else
- autoCommit = true;
- }
- catch (IOException e) {
- throw new BinaryObjectException(e);
- }
+ if (ver.compareTo(VER_2_4_0) >= 0)
+ lastStreamBatch = reader.readBoolean();
+
+ if (ver.compareTo(VER_2_7_0) >= 0)
+ autoCommit = reader.readBoolean();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteResult.java
index 3fc9dd7..0d93244e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteResult.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -97,8 +98,9 @@ public class JdbcBatchExecuteResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeInt(errCode);
writer.writeString(errMsg);
@@ -107,8 +109,9 @@ public class JdbcBatchExecuteResult extends JdbcResult {
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
errCode = reader.readInt();
errMsg = reader.readString();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBulkLoadAckResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBulkLoadAckResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBulkLoadAckResult.java
index e670baf..b0750fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBulkLoadAckResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBulkLoadAckResult.java
@@ -21,6 +21,7 @@ import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
import org.apache.ignite.internal.processors.bulkload.BulkLoadAckClientParameters;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.sql.command.SqlBulkLoadCommand;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -81,8 +82,9 @@ public class JdbcBulkLoadAckResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeLong(qryId);
writer.writeString(params.localFileName());
@@ -90,8 +92,9 @@ public class JdbcBulkLoadAckResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
qryId = reader.readLong();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBulkLoadBatchRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBulkLoadBatchRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBulkLoadBatchRequest.java
index 7db4951..347a5df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBulkLoadBatchRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBulkLoadBatchRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.sql.command.SqlBulkLoadCommand;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.NotNull;
@@ -143,8 +144,9 @@ public class JdbcBulkLoadBatchRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeLong(qryId);
writer.writeInt(batchIdx);
@@ -153,8 +155,9 @@ public class JdbcBulkLoadBatchRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
qryId = reader.readLong();
batchIdx = reader.readInt();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMeta.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMeta.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMeta.java
index 5b6304d..2b08fb4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMeta.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMeta.java
@@ -21,6 +21,7 @@ import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
import org.apache.ignite.internal.jdbc.thin.JdbcThinUtils;
import org.apache.ignite.internal.jdbc2.JdbcUtils;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -158,7 +159,8 @@ public class JdbcColumnMeta implements JdbcRawBinarylizable {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) {
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) {
writer.writeString(schemaName);
writer.writeString(tblName);
writer.writeString(colName);
@@ -169,7 +171,8 @@ public class JdbcColumnMeta implements JdbcRawBinarylizable {
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) {
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) {
schemaName = reader.readString();
tblName = reader.readString();
colName = reader.readString();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV2.java
index a2b4acf..6c77b55 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV2.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -54,15 +55,17 @@ public class JdbcColumnMetaV2 extends JdbcColumnMeta{
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) {
+ super.writeBinary(writer, ver);
writer.writeBoolean(nullable);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) {
+ super.readBinary(reader, ver);
nullable = reader.readBoolean();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV3.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV3.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV3.java
index 9911be0..8f8adfe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV3.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV3.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -63,15 +64,17 @@ public class JdbcColumnMetaV3 extends JdbcColumnMetaV2 {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) {
+ super.writeBinary(writer, ver);
writer.writeString(dfltValue);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) {
+ super.readBinary(reader, ver);
dfltValue = reader.readString();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV4.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV4.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV4.java
index ec76983..6bdc597 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV4.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcColumnMetaV4.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -68,16 +69,18 @@ public class JdbcColumnMetaV4 extends JdbcColumnMetaV3 {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) {
+ super.writeBinary(writer, ver);
writer.writeInt(precision);
writer.writeInt(scale);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) {
+ super.readBinary(reader, ver);
precision = reader.readInt();
scale = reader.readInt();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
index 9f3fe2a..5e9a1b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
@@ -142,7 +142,7 @@ public class JdbcConnectionContext extends ClientListenerAbstractConnectionConte
if (ver.compareTo(VER_2_3_0) >= 0)
skipReducerOnUpdate = reader.readBoolean();
- if (ver.compareTo(VER_2_5_0) >= 0) {
+ if (ver.compareTo(VER_2_7_0) >= 0) {
String nestedTxModeName = reader.readString();
if (!F.isEmpty(nestedTxModeName)) {
@@ -153,7 +153,9 @@ public class JdbcConnectionContext extends ClientListenerAbstractConnectionConte
throw new IgniteCheckedException("Invalid nested transactions handling mode: " + nestedTxModeName);
}
}
+ }
+ if (ver.compareTo(VER_2_5_0) >= 0) {
String user = null;
String passwd = null;
@@ -170,7 +172,7 @@ public class JdbcConnectionContext extends ClientListenerAbstractConnectionConte
actx = authenticate(user, passwd);
}
- parser = new JdbcMessageParser(ctx);
+ parser = new JdbcMessageParser(ctx, ver);
JdbcResponseSender sender = new JdbcResponseSender() {
@Override public void send(ClientListenerResponse resp) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcIndexMeta.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcIndexMeta.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcIndexMeta.java
index d33f887..d7de6d7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcIndexMeta.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcIndexMeta.java
@@ -24,6 +24,7 @@ import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.cache.QueryIndexType;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.processors.query.GridQueryIndexDescriptor;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -123,7 +124,8 @@ public class JdbcIndexMeta implements JdbcRawBinarylizable {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
writer.writeString(schemaName);
writer.writeString(tblName);
writer.writeString(idxName);
@@ -142,7 +144,8 @@ public class JdbcIndexMeta implements JdbcRawBinarylizable {
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
schemaName = reader.readString();
tblName = reader.readString();
idxName = reader.readString();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMessageParser.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMessageParser.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMessageParser.java
index 7a9c2ab..1718c00 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMessageParser.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMessageParser.java
@@ -24,6 +24,7 @@ import org.apache.ignite.internal.binary.streams.BinaryHeapInputStream;
import org.apache.ignite.internal.binary.streams.BinaryHeapOutputStream;
import org.apache.ignite.internal.binary.streams.BinaryInputStream;
import org.apache.ignite.internal.processors.odbc.ClientListenerMessageParser;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.processors.odbc.ClientListenerRequest;
import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
@@ -34,14 +35,20 @@ public class JdbcMessageParser implements ClientListenerMessageParser {
/** Kernal context. */
private final GridKernalContext ctx;
+ /** Client protocol version. */
+ private final ClientListenerProtocolVersion ver;
+
/** Initial output stream capacity. */
protected static final int INIT_CAP = 1024;
/**
* @param ctx Context.
+ * @param ver Client protocol version.
*/
- public JdbcMessageParser(GridKernalContext ctx) {
+ public JdbcMessageParser(GridKernalContext ctx,
+ ClientListenerProtocolVersion ver) {
this.ctx = ctx;
+ this.ver = ver;
}
/**
@@ -68,7 +75,7 @@ public class JdbcMessageParser implements ClientListenerMessageParser {
BinaryReaderExImpl reader = createReader(msg);
- return JdbcRequest.readRequest(reader);
+ return JdbcRequest.readRequest(reader, ver);
}
/** {@inheritDoc} */
@@ -81,7 +88,7 @@ public class JdbcMessageParser implements ClientListenerMessageParser {
BinaryWriterExImpl writer = createWriter(INIT_CAP);
- res.writeBinary(writer);
+ res.writeBinary(writer, ver);
return writer.array();
}}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsRequest.java
index fca1bf7..389629e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.Nullable;
@@ -78,8 +79,9 @@ public class JdbcMetaColumnsRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeString(schemaName);
writer.writeString(tblName);
@@ -87,8 +89,9 @@ public class JdbcMetaColumnsRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
schemaName = reader.readString();
tblName = reader.readString();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsResult.java
index 9931ce0..199c76d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaColumnsResult.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -77,8 +78,9 @@ public class JdbcMetaColumnsResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
if (F.isEmpty(meta))
writer.writeInt(0);
@@ -86,13 +88,14 @@ public class JdbcMetaColumnsResult extends JdbcResult {
writer.writeInt(meta.size());
for(JdbcColumnMeta m : meta)
- m.writeBinary(writer);
+ m.writeBinary(writer, ver);
}
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
int size = reader.readInt();
@@ -104,7 +107,7 @@ public class JdbcMetaColumnsResult extends JdbcResult {
for (int i = 0; i < size; ++i) {
JdbcColumnMeta m = createMetaColumn();
- m.readBinary(reader);
+ m.readBinary(reader, ver);
meta.add(m);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaIndexesRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaIndexesRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaIndexesRequest.java
index d4a53d8..28fe558 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaIndexesRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaIndexesRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.Nullable;
@@ -66,16 +67,18 @@ public class JdbcMetaIndexesRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeString(schemaName);
writer.writeString(tblName);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
schemaName = reader.readString();
tblName = reader.readString();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaIndexesResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaIndexesResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaIndexesResult.java
index 2316dfc..b180954 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaIndexesResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaIndexesResult.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -57,8 +58,9 @@ public class JdbcMetaIndexesResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
if (F.isEmpty(meta))
writer.writeInt(0);
@@ -66,13 +68,14 @@ public class JdbcMetaIndexesResult extends JdbcResult {
writer.writeInt(meta.size());
for(JdbcIndexMeta m : meta)
- m.writeBinary(writer);
+ m.writeBinary(writer, ver);
}
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
int size = reader.readInt();
@@ -84,7 +87,7 @@ public class JdbcMetaIndexesResult extends JdbcResult {
for (int i = 0; i < size; ++i) {
JdbcIndexMeta m = new JdbcIndexMeta();
- m.readBinary(reader);
+ m.readBinary(reader, ver);
meta.add(m);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaParamsRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaParamsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaParamsRequest.java
index 6b955f9..360f17d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaParamsRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaParamsRequest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.odbc.jdbc;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -65,16 +66,18 @@ public class JdbcMetaParamsRequest extends JdbcRequest {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
writer.writeString(schemaName);
writer.writeString(sql);
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
schemaName = reader.readString();
sql = reader.readString();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469fb49e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaParamsResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaParamsResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaParamsResult.java
index 7563e01..43c2422 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaParamsResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcMetaParamsResult.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.internal.binary.BinaryReaderExImpl;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -49,8 +50,9 @@ public class JdbcMetaParamsResult extends JdbcResult {
}
/** {@inheritDoc} */
- @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
- super.writeBinary(writer);
+ @Override public void writeBinary(BinaryWriterExImpl writer,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.writeBinary(writer, ver);
if (F.isEmpty(meta))
writer.writeInt(0);
@@ -58,13 +60,14 @@ public class JdbcMetaParamsResult extends JdbcResult {
writer.writeInt(meta.size());
for(JdbcParameterMeta m : meta)
- m.writeBinary(writer);
+ m.writeBinary(writer, ver);
}
}
/** {@inheritDoc} */
- @Override public void readBinary(BinaryReaderExImpl reader) throws BinaryObjectException {
- super.readBinary(reader);
+ @Override public void readBinary(BinaryReaderExImpl reader,
+ ClientListenerProtocolVersion ver) throws BinaryObjectException {
+ super.readBinary(reader, ver);
int size = reader.readInt();
@@ -76,7 +79,7 @@ public class JdbcMetaParamsResult extends JdbcResult {
for (int i = 0; i < size; ++i) {
JdbcParameterMeta m = new JdbcParameterMeta();
- m.readBinary(reader);
+ m.readBinary(reader, ver);
meta.add(m);
}
[28/50] [abbrv] ignite git commit: IGNITE-9360 Remove SnapTreeMap and
tests on it. - Fixes #4608.
Posted by ag...@apache.org.
IGNITE-9360 Remove SnapTreeMap and tests on it. - Fixes #4608.
Signed-off-by: Dmitriy Govorukhin <dm...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/35706b94
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/35706b94
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/35706b94
Branch: refs/heads/ignite-5960
Commit: 35706b94e4d714f426eee12cd2c3d9a1e8452dad
Parents: 9bfc823
Author: Ilya Kasnacheev <il...@gmail.com>
Authored: Tue Sep 4 18:49:06 2018 +0300
Committer: Dmitriy Govorukhin <dm...@gmail.com>
Committed: Tue Sep 4 18:49:06 2018 +0300
----------------------------------------------------------------------
.../util/snaptree/CopyOnWriteManager.java | 390 ---
.../ignite/internal/util/snaptree/Epoch.java | 131 -
.../internal/util/snaptree/EpochNode.java | 433 ---
.../internal/util/snaptree/SnapTreeMap.java | 2917 ------------------
.../internal/util/snaptree/package-info.java | 22 -
.../testsuites/IgniteUtilSelfTestSuite.java | 2 -
.../apache/ignite/util/GridIndexFillTest.java | 259 --
.../loadtests/h2indexing/GridTestEntity.java | 67 -
.../loadtests/h2indexing/GridTreeBenchmark.java | 280 --
9 files changed, 4501 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/35706b94/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/CopyOnWriteManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/CopyOnWriteManager.java b/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/CopyOnWriteManager.java
deleted file mode 100644
index 685668a..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/CopyOnWriteManager.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Copyright (c) 2009 Stanford University, unless otherwise specified.
- * All rights reserved.
- *
- * This software was developed by the Pervasive Parallelism Laboratory of
- * Stanford University, California, USA.
- *
- * Permission to use, copy, modify, and distribute this software in source
- * or binary form for any purpose with or without fee is hereby granted,
- * provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * 3. Neither the name of Stanford University nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-package org.apache.ignite.internal.util.snaptree;
-
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.AbstractQueuedSynchronizer;
-
-/** Manages copy-on-write behavior for a concurrent tree structure. It is
- * assumed that the managed structure allows concurrent mutation, but that no
- * mutating operations may be active when a copy-on-write snapshot of tree is
- * taken. Because it is difficult to update the size of data structure in a
- * highly concurrent fashion, the <code>CopyOnWriteManager</code> also manages
- * a running total that represents the size of the contained tree structure.
- * <p>
- * Users should implement the {@link #freezeAndClone(Object)} and
- * {@link #cloneFrozen(Object)} methods.
- */
-@SuppressWarnings("ALL")
-abstract public class CopyOnWriteManager<E> implements Cloneable {
-
- /** This is basically a stripped-down CountDownLatch. Implementing our own
- * reduces the object count by one, and it gives us access to the
- * uninterruptable acquireShared.
- */
- private class Latch extends AbstractQueuedSynchronizer {
- /** */
- private static final long serialVersionUID = 0L;
-
- Latch(final boolean triggered) {
- setState(triggered ? 0 : 1);
- }
-
- public int tryAcquireShared(final int acquires) {
- // 1 = success, and followers may also succeed
- // -1 = failure
- return getState() == 0 ? 1 : -1;
- }
-
- public boolean tryReleaseShared(final int releases) {
- // Before, state is either 0 or 1. After, state is always 0.
- return compareAndSetState(1, 0);
- }
- }
-
- private static final int MUTATE = 1;
- private static final int MUTATE_AFTER_FREEZE = 2;
- private static final int BULK_READ = 3;
- private static final int BULK_READ_AFTER_FREEZE = 4;
-
- private class COWEpoch extends EpochNode {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Tripped after this COWEpoch is installed as active. */
- private final Latch _activated;
-
- /** True iff this is a mutating epoch. */
- final boolean mutationAllowed;
-
- /** The value used by this epoch. */
- E value;
-
- /** The computed size of <code>value</code>, as of the beginning of
- * this epoch.
- */
- int initialSize;
-
- /** A frozen E equal to <code>value</code>, if not <code>dirty</code>. */
- private volatile E _frozenValue;
-
- /** True if any mutations have been performed on <code>value</code>. */
- volatile boolean dirty;
-
- /** The epoch that will follow this one, created on demand. */
- final AtomicReference<COWEpoch> successorRef = new AtomicReference<COWEpoch>(null);
-
- /** A ticket on the successor, released when this epoch is closed. */
- Epoch.Ticket successorTicket;
-
- /** True if the successor should freeze and clone this epoch's value. */
- boolean freezeRequested;
-
- private COWEpoch(final boolean mutationAllowed) {
- this._activated = new Latch(false);
- this.mutationAllowed = mutationAllowed;
- }
-
- public COWEpoch(final E value, final E frozenValue, final int initialSize) {
- this._activated = new Latch(true); // pre-triggered
- this.mutationAllowed = true;
- this.value = value;
- this.initialSize = initialSize;
- this._frozenValue = frozenValue;
- this.dirty = frozenValue == null;
- }
-
- EpochNode attemptInitialArrive() {
- return super.attemptArrive();
- }
-
- @Override
- public EpochNode attemptArrive() {
- final EpochNode ticket = super.attemptArrive();
- if (ticket != null && !dirty) {
- dirty = true;
- _frozenValue = null;
- }
- return ticket;
- }
-
- private void setFrozenValue(final E v) {
- if (!dirty) {
- _frozenValue = v;
- if (dirty) {
- _frozenValue = null;
- }
- }
- }
-
- E getFrozenValue() {
- final E v = _frozenValue;
- return dirty ? null : v;
- }
-
- protected void onClosed(final int dataSum) {
- assert(dataSum == 0 || dirty);
-
- final COWEpoch succ = successorRef.get();
- if (freezeRequested) {
- succ.value = freezeAndClone(value);
- succ.setFrozenValue(value);
- }
- else {
- succ.value = value;
- if (dirty) {
- succ.dirty = true;
- }
- else {
- succ.setFrozenValue(_frozenValue);
- }
- }
- succ.initialSize = initialSize + dataSum;
-
- _active = succ;
- successorTicket.leave(0);
- succ._activated.releaseShared(1);
- }
-
- public void awaitActivated() {
- _activated.acquireShared(1);
- }
-
- public COWEpoch getOrCreateSuccessor(final boolean preferredMutation) {
- final COWEpoch existing = successorRef.get();
- if (existing != null) {
- return existing;
- }
-
- final COWEpoch repl = new COWEpoch(preferredMutation);
- if (attemptInstallSuccessor(repl)) {
- return repl;
- }
-
- return successorRef.get();
- }
-
- public boolean attemptInstallSuccessor(final COWEpoch succ) {
- final Epoch.Ticket t = succ.attemptInitialArrive();
- if (successorRef.compareAndSet(null, succ)) {
- successorTicket = t;
- beginClose();
- return true;
- }
- else {
- return false;
- }
- }
- }
-
- private volatile COWEpoch _active;
-
- /** Creates a new {@link CopyOnWriteManager} holding
- * <code>initialValue</code>, with an assumed size of
- * <code>initialSize</code>.
- */
- public CopyOnWriteManager(final E initialValue, final int initialSize) {
- _active = new COWEpoch(initialValue, null, initialSize);
- }
-
- /** The implementing method must mark <code>value</code> as shared, and
- * return a new object to use in its place. Hopefully, the majority of
- * the work of the clone can be deferred by copy-on-write.
- */
- abstract protected E freezeAndClone(final E value);
-
- /** Returns a clone of a frozen E. */
- abstract protected E cloneFrozen(E frozenValue);
-
- public CopyOnWriteManager<E> clone() {
- final CopyOnWriteManager<E> copy;
- try {
- copy = (CopyOnWriteManager<E>) super.clone();
- }
- catch (final CloneNotSupportedException xx) {
- throw new Error("unexpected", xx);
- }
-
- COWEpoch a = _active;
- E f = a.getFrozenValue();
- while (f == null) {
- a.freezeRequested = true;
- final COWEpoch succ = a.getOrCreateSuccessor(a.mutationAllowed);
- succ.awaitActivated();
- if (a.value != succ.value) {
- f = a.value;
- }
- a = succ;
- }
-
- copy.createNewEpoch(f, a);
- return copy;
- }
-
- private void createNewEpoch(E f, COWEpoch a)
- {
- _active = new COWEpoch(cloneFrozen(f), f, a.initialSize);
- }
-
- /** Returns a reference to the tree structure suitable for a read
- * operation. The returned structure may be mutated by operations that
- * have the permission of this {@link CopyOnWriteManager}, but they will
- * not observe changes managed by other instances.
- */
- public E read() {
- return _active.value;
- }
-
- /** Obtains permission to mutate the copy-on-write value held by this
- * instance, perhaps blocking while a concurrent snapshot is being
- * performed. {@link Epoch.Ticket#leave} must be called exactly once on
- * the object returned from this method, after the mutation has been
- * completed. The change in size reflected by the mutation should be
- * passed as the parameter to <code>leave</code>.
- */
- public Epoch.Ticket beginMutation() {
- return begin(true);
- }
-
- public Epoch.Ticket beginQuiescent() {
- return begin(false);
- }
-
- private Epoch.Ticket begin(final boolean mutation) {
- final COWEpoch active = _active;
- if (active.mutationAllowed == mutation) {
- final Epoch.Ticket ticket = active.attemptArrive();
- if (ticket != null) {
- return ticket;
- }
- }
- return begin(mutation, active);
- }
-
- private Epoch.Ticket begin(final boolean mutation, COWEpoch epoch) {
- while (true) {
- COWEpoch succ = epoch.successorRef.get();
- if (succ == null) {
- final COWEpoch newEpoch = new COWEpoch(mutation);
- final Epoch.Ticket newTicket = newEpoch.attemptArrive();
- if (epoch.attemptInstallSuccessor(newEpoch)) {
- // can't use the ticket until the new epoch is activated
- newEpoch.awaitActivated();
- return newTicket;
- }
-
- // if our CAS failed, somebody else succeeded
- succ = epoch.successorRef.get();
- }
-
- // is the successor created by someone else suitable?
- if (succ.mutationAllowed == mutation) {
- final Epoch.Ticket ticket = succ.attemptArrive();
- if (ticket != null) {
- succ.awaitActivated();
- return ticket;
- }
- }
-
- epoch = succ;
- }
- }
-
- /** Returns a reference to the tree structure suitable for a mutating
- * operation. This method may only be called under the protection of a
- * ticket returned from {@link #beginMutation}.
- */
- public E mutable() {
- return _active.value;
- }
-
- /** Returns a reference to a snapshot of this instance's tree structure
- * that may be read, but not written. This is accomplished by suspending
- * mutation, replacing the mutable root of this manager with the result of
- * <code>freezeAndClone(root, false)</code>, and then returning a
- * reference to the old root. Successive calls to this method may return
- * the same instance.
- */
- public E frozen() {
- COWEpoch a = _active;
- E f = a.getFrozenValue();
- while (f == null) {
- a.freezeRequested = true;
- final COWEpoch succ = a.getOrCreateSuccessor(a.mutationAllowed);
- succ.awaitActivated();
- if (a.value != succ.value) {
- f = a.value;
- }
- a = succ;
- }
- return f;
- }
-
- /** Returns a reference to a snapshot of this instance's tree structure,
- * if one is available without requiring any additional copying, otherwise
- * returns null. May be used in combination with {@link #beginQuiescent}
- * to perform quiescent reads with minimal cost.
- */
- public E availableFrozen() {
- return _active.getFrozenValue();
- }
-
- /** Returns true if the computed {@link #size} is zero. */
- public boolean isEmpty() {
- // for a different internal implementation (such as a C-SNZI) we might
- // be able to do better than this
- return size() == 0;
- }
-
- /** Returns the sum of the <code>initialSize</code> parameter passed to the
- * constructor, and the size deltas passed to {@link Epoch.Ticket#leave}
- * for all of the mutation tickets. The result returned is linearizable
- * with mutations, which requires mutation to be quiesced. No tree freeze
- * is required, however.
- */
- public int size() {
- final COWEpoch a = _active;
- final Integer delta = a.attemptDataSum();
- if (delta != null) {
- return a.initialSize + delta;
- }
-
- // wait for an existing successor, or force one if not already in progress
- final COWEpoch succ = a.getOrCreateSuccessor(a.mutationAllowed);
- succ.awaitActivated();
- return succ.initialSize;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/35706b94/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/Epoch.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/Epoch.java b/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/Epoch.java
deleted file mode 100644
index c85320d..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/Epoch.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright (c) 2009 Stanford University, unless otherwise specified.
- * All rights reserved.
- *
- * This software was developed by the Pervasive Parallelism Laboratory of
- * Stanford University, California, USA.
- *
- * Permission to use, copy, modify, and distribute this software in source
- * or binary form for any purpose with or without fee is hereby granted,
- * provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * 3. Neither the name of Stanford University nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-package org.apache.ignite.internal.util.snaptree;
-
-/** A <code>Epoch</code> has a lifecycle consisting of three phases: active,
- * closing, and closed. During the active phase partipants may arrive and
- * leave the epoch. Once a close has been requested, new participants are not
- * allowed, only leaving is possible. Once close has been requested and all
- * participants have left, the epoch is transitioned to the closed state.
- * <p>
- * Entry is performed with {@link #attemptArrive}, which returns a non-null
- * ticket on success or null if {@link #beginClose} has already been called.
- * Each successful call to <code>attemptArrive</code> must be paired by a call
- * to {@link Ticket#leave} on the returned ticket.
- * <p>
- * The abstract method {@link #onClosed} will be invoked exactly once after
- * the epoch becomes closed. It will be passed the sum of the values passed
- * to {@link Ticket#leave}. There is no way to query the current participant
- * count or state of the epoch without changing it.
- * <p>
- * Internally the epoch responds to contention by increasing its size,
- * striping the participant count across multiple objects (and hopefully
- * multiple cache lines). Once close has begun, the epoch converts itself to
- * a single-shot hierarchical barrier, that also performs a hierarchical
- * reduction of the leave parameters.
- */
-@SuppressWarnings("ALL")
-abstract public class Epoch {
-
- /** Represents a single successful arrival to an {@link Epoch}. */
- public interface Ticket {
- /** Informs the epoch that returned this ticket that the participant
- * has left. This method should be called exactly once per ticket.
- * The sum of the <code>data</code> values for all tickets will be
- * computed and passed to {@link Epoch#onClosed}.
- */
- void leave(int data);
- }
-
- private final Root _root = new Root();
-
- /** Returns a {@link Ticket} indicating a successful arrival, if no call to
- * {@link #beginClose} has been made for this epoch, or returns null if
- * close has already begun. {@link Ticket#leave} must be called exactly
- * once on any returned ticket.
- */
- public Ticket attemptArrive() {
- return _root.attemptArrive();
- }
-
- /** Prevents new arrivals from succeeding, then returns immediately.
- * {@link #onClosed} will be called after all outstanding tickets have
- * been returned. To block until close is complete, add some sort of
- * synchronization logic to the user-defined implementation of {@link
- * #onClosed}.
- */
- public void beginClose() {
- _root.beginClose();
- }
-
- /** Override this method to provide user-defined behavior.
- * <code>dataSum</code> will be the sum of the <code>data</code> values
- * passed to {@link Ticket#leave} for all tickets in this epoch.
- * <p>
- * As a simple example, a blocking close operation may be defined by:<pre>
- * class BlockingEpoch extends Epoch {
- * private final CountDownLatch _closed = new CountDownLatch(1);
- *
- * public void blockingClose() throws InterruptedException {
- * beginClose();
- * _closed.await();
- * }
- *
- * protected void onClosed(int dataSum) {
- * _closed.countDown(1);
- * }
- * }
- * </pre>
- */
- abstract protected void onClosed(int dataSum);
-
- //////////////// debugging stuff
-
- int computeSpread() {
- return _root.computeSpread();
- }
-
- //////////////// internal implementation
-
- private class Root extends EpochNode {
- /** */
- private static final long serialVersionUID = 0L;
-
- protected void onClosed(final int dataSum) {
- Epoch.this.onClosed(dataSum);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/35706b94/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/EpochNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/EpochNode.java b/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/EpochNode.java
deleted file mode 100644
index 7ceec45..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/EpochNode.java
+++ /dev/null
@@ -1,433 +0,0 @@
-/*
- * Copyright (c) 2009 Stanford University, unless otherwise specified.
- * All rights reserved.
- *
- * This software was developed by the Pervasive Parallelism Laboratory of
- * Stanford University, California, USA.
- *
- * Permission to use, copy, modify, and distribute this software in source
- * or binary form for any purpose with or without fee is hereby granted,
- * provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * 3. Neither the name of Stanford University nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-package org.apache.ignite.internal.util.snaptree;
-
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
-
-/** Provides an implementation of the behavior of an {@link Epoch}. */
-@SuppressWarnings("ALL")
-abstract class EpochNode extends AtomicLong implements Epoch.Ticket {
-
- private static final int TRIES_BEFORE_SUBTREE = 2;
- private static final int CLOSER_HEAD_START = 1000;
-
- /** This includes the root. 7 or fewer procs gets 2, 63 or fewer gets
- * 3, 511 or fewer 4. We observe that the node count reported by {@link
- * #computeSpread} is roughly twice the number of hardware contexts in
- * use.
- */
- private static final int MAX_LEVELS = 2 + log8(Runtime.getRuntime().availableProcessors());
-
- /** Returns floor(log_base_8(value)). */
- private static int log8(final int value) {
- return (31 - Integer.numberOfLeadingZeros(value)) / 3;
- }
-
- //////////////// branching factor
-
- private static final int LOG_BF = 3;
- private static final int BF = 1 << LOG_BF;
- private static final int BF_MASK = BF - 1;
-
- //////////////// bit packing
-
- private static final int DATA_SUM_SHIFT = 32;
- private static int dataSum(long state) { return (int)(state >> DATA_SUM_SHIFT); }
- private static long withDataDelta(long state, int delta) { return state + (((long) delta) << DATA_SUM_SHIFT); }
-
- private static final int CHILD_CLOSED_SHIFT = 32 - BF;
- private static long ALL_CHILDREN_CLOSED = ((1L << BF) - 1L) << CHILD_CLOSED_SHIFT;
- private static long childClosedBit(int which) { return 1L << (CHILD_CLOSED_SHIFT + which); }
- private static boolean isChildClosed(long state, int which) { return (state & childClosedBit(which)) != 0; }
- private static long withChildClosed(long state, int which, long childState) {
- assert(!isChildClosed(state, which));
- return withDataDelta(state | childClosedBit(which), dataSum(childState));
- }
- private static boolean isAllChildrenClosed(long state) { return (state & ALL_CHILDREN_CLOSED) == ALL_CHILDREN_CLOSED; }
-
- private static final int CHILD_PRESENT_SHIFT = CHILD_CLOSED_SHIFT - BF;
- private static final long ANY_CHILD_PRESENT = ((1L << BF) - 1L) << CHILD_PRESENT_SHIFT;
- private static long childPresentBit(int which) { return 1L << (CHILD_PRESENT_SHIFT + which); }
- private static boolean isChildPresent(long state, int which) { return (state & childPresentBit(which)) != 0; }
- private static long withChildPresent(long state, int which) { return state | childPresentBit(which); }
- private static boolean isAnyChildPresent(long state) { return (state & ANY_CHILD_PRESENT) != 0; }
-
- private static final long MARK = (1L << (CHILD_PRESENT_SHIFT - 1));
- private static boolean isMarked(long state) { return (state & MARK) != 0L; }
- /** Records all non-present children as closed. */
- private static long withMarked(long state) {
- final int missingChildren = (~((int) state) >> CHILD_PRESENT_SHIFT) & ((1 << BF) - 1);
- return state | MARK | (((long) missingChildren) << CHILD_CLOSED_SHIFT);
- }
-
- private static final long ENTRY_COUNT_MASK = MARK - 1;
- private static int entryCount(long state) { return (int) (state & ENTRY_COUNT_MASK); }
- private static long withArrive(long state) { return state + 1; }
- private static long withLeave(long state, int dataDelta) { return withDataDelta(state - 1, dataDelta); }
- private static boolean mayArrive(long state) { return entryCount(state) != ENTRY_COUNT_MASK; }
- private static boolean mayLeave(long state) { return entryCount(state) != 0; }
-
- private static final long CLOSED_MASK = MARK | ALL_CHILDREN_CLOSED | ENTRY_COUNT_MASK;
- private static final long CLOSED_VALUE = MARK | ALL_CHILDREN_CLOSED;
- private static boolean isClosed(long state) { return (state & CLOSED_MASK) == CLOSED_VALUE; }
-
- private static final long ENTRY_FAST_PATH_MASK = ANY_CHILD_PRESENT | MARK | (ENTRY_COUNT_MASK - (ENTRY_COUNT_MASK >> 1));
- /** Not marked, no children, and no overflow possible. */
- private static boolean isEntryFastPath(long state) { return (state & ENTRY_FAST_PATH_MASK) == 0L; }
-
- //////////////// subclasses
-
- private static class Child extends EpochNode {
- /** */
- private static final long serialVersionUID = 0L;
-
- private Child(final EpochNode parent, final int whichInParent) {
- super(parent, whichInParent);
- }
-
- protected void onClosed(final int dataSum) {
- throw new Error();
- }
- }
-
- //////////////// instance state
-
- private static final AtomicReferenceFieldUpdater[] childrenUpdaters = {
- AtomicReferenceFieldUpdater.newUpdater(EpochNode.class, EpochNode.class, "_child0"),
- AtomicReferenceFieldUpdater.newUpdater(EpochNode.class, EpochNode.class, "_child1"),
- AtomicReferenceFieldUpdater.newUpdater(EpochNode.class, EpochNode.class, "_child2"),
- AtomicReferenceFieldUpdater.newUpdater(EpochNode.class, EpochNode.class, "_child3"),
- AtomicReferenceFieldUpdater.newUpdater(EpochNode.class, EpochNode.class, "_child4"),
- AtomicReferenceFieldUpdater.newUpdater(EpochNode.class, EpochNode.class, "_child5"),
- AtomicReferenceFieldUpdater.newUpdater(EpochNode.class, EpochNode.class, "_child6"),
- AtomicReferenceFieldUpdater.newUpdater(EpochNode.class, EpochNode.class, "_child7")
- };
-
- private final EpochNode _parent;
- private final int _whichInParent;
-
- // It would be cleaner to use an array of children, but we want to force
- // all of the bulk into the same object as the AtomicLong.value.
-
- // To avoid races between creating a child and marking a node as closed,
- // we add a bit to the state for each child that records whether it
- // *should* exist. If we find that the bit is set but a child is missing,
- // we can create it ourself.
-
- private volatile EpochNode _child0;
- private volatile EpochNode _child1;
- private volatile EpochNode _child2;
- private volatile EpochNode _child3;
- private volatile EpochNode _child4;
- private volatile EpochNode _child5;
- private volatile EpochNode _child6;
- private volatile EpochNode _child7;
-
- EpochNode() {
- _parent = null;
- _whichInParent = 0;
- }
-
- private EpochNode(final EpochNode parent, final int whichInParent) {
- _parent = parent;
- _whichInParent = whichInParent;
- }
-
- //////////////// provided by the caller
-
- abstract protected void onClosed(int dataSum);
-
- //////////////// child management
-
- private EpochNode getChildFromField(final int which) {
- switch (which) {
- case 0: return _child0;
- case 1: return _child1;
- case 2: return _child2;
- case 3: return _child3;
- case 4: return _child4;
- case 5: return _child5;
- case 6: return _child6;
- case 7: return _child7;
- default: return null;
- }
- }
-
- private EpochNode getChild(final long state, final int which) {
- if (!isChildPresent(state, which)) {
- return null;
- }
- final EpochNode existing = getChildFromField(which);
- return (existing != null) ? existing : constructPresentChild(which);
- }
-
- @SuppressWarnings("unchecked")
- private EpochNode constructPresentChild(final int which) {
- final EpochNode n = new Child(this, which);
- return childrenUpdaters[which].compareAndSet(this, null, n) ? n : getChildFromField(which);
- }
-
- private EpochNode getOrCreateChild(final int which) {
- final EpochNode existing = getChildFromField(which);
- return (existing != null) ? existing : createChild(which);
- }
-
- private EpochNode createChild(final int which) {
- while (true) {
- final long state = get();
- if (isMarked(state)) {
- // whatever we've got is what we've got
- return getChild(state, which);
- }
- if (compareAndSet(state, withChildPresent(state, which))) {
- // the child now should exist, but we must still actually
- // construct and link in the instance
- return constructPresentChild(which);
- }
- }
- }
-
- /** Returns the <code>Node</code> to decr on success, null if
- * {@link #beginClose} has already been called on this instance.
- */
- public EpochNode attemptArrive() {
- final long state = get();
- if (isEntryFastPath(state) && compareAndSet(state, withArrive(state))) {
- return this;
- }
- else {
- return attemptArrive(0, 1);
- }
- }
-
- private int getIdentity() {
- final int h = System.identityHashCode(Thread.currentThread());
-
- // Multiply by -127, as suggested by java.util.IdentityHashMap.
- // We also set an bit we don't use, to make sure it is never zero.
- return (h - (h << 7)) | (1 << 31);
- }
-
- /** level 1 is the root. */
- private EpochNode attemptArrive(int id, final int level) {
- int tries = 0;
- while (true) {
- final long state = get();
- if (isMarked(state)) {
- return null;
- }
- if (isAnyChildPresent(state) ||
- (tries >= TRIES_BEFORE_SUBTREE && level < MAX_LEVELS)) {
- // Go deeper if we have previously detected contention, or if
- // we are currently detecting it. Lazy computation of our
- // current identity.
- if (id == 0) {
- id = getIdentity();
- }
- final EpochNode child = getOrCreateChild(id & BF_MASK);
- if (child == null) {
- return null;
- }
- return child.attemptArrive(id >> LOG_BF, level + 1);
- }
- if (!mayArrive(state)) {
- throw new IllegalStateException("maximum arrival count of " + ENTRY_COUNT_MASK + " exceeded");
- }
- if (compareAndSet(state, withArrive(state))) {
- // success
- return this;
- }
-
- ++tries;
- }
- }
-
- /** Should be called on every non-null return value from attemptArrive. */
- public void leave(final int dataDelta) {
- while (true) {
- final long state = get();
- if (!mayLeave(state)) {
- throw new IllegalStateException("incorrect call to Epoch.leave");
- }
- final long after = withLeave(state, dataDelta);
- if (compareAndSet(state, after)) {
- if (isClosed(after)) {
- newlyClosed(after);
- }
- return;
- }
- }
- }
-
- private void newlyClosed(final long state) {
- if (_parent != null) {
- // propogate
- _parent.childIsNowClosed(_whichInParent, state);
- }
- else {
- // report
- onClosed(dataSum(state));
- }
- }
-
- private void childIsNowClosed(final int which, final long childState) {
- while (true) {
- final long state = get();
- if (isChildClosed(state, which)) {
- // not our problem
- return;
- }
- final long after = withChildClosed(state, which, childState);
- if (compareAndSet(state, after)) {
- if (isClosed(after)) {
- newlyClosed(after);
- }
- return;
- }
- }
- }
-
- /** Prevents subsequent calls to {@link #attemptArrive} from succeeding. */
- public void beginClose() {
- int attempts = 0;
- long state;
- while (true) {
- ++attempts;
-
- state = get();
- if (isClosed(state)) {
- return;
- }
-
- if (isMarked(state)) {
- // give the thread that actually performed this transition a
- // bit of a head start
- if (attempts < CLOSER_HEAD_START) {
- continue;
- }
- break;
- }
-
- // every child that is not present will be recorded as closed by withMarked
- final long after = withMarked(state);
- if (compareAndSet(state, after)) {
- if (isAllChildrenClosed(after)) {
- if (isClosed(after) && _parent == null) {
- // finished in one CAS, yeah!
- onClosed(dataSum(after));
- }
- // no second stage necessary
- return;
- }
- // CAS successful, so now we need to beginClose() the children
- break;
- }
- }
-
- // no new child bits can be set after marking, so this gets everyone
- for (int which = 0; which < BF; ++which) {
- final EpochNode child = getChild(state, which);
- if (child != null) {
- child.beginClose();
- }
- }
-
- // Rather than have each child bubble up its closure, we gather it
- // here to reduce the number of CASs required.
- while (true) {
- final long before = get();
- long after = before;
- for (int which = 0; which < BF; ++which) {
- if (!isChildClosed(before, which)) {
- final long childState = getChildFromField(which).get();
- if (isClosed(childState)) {
- after = withChildClosed(after, which, childState);
- }
- }
- }
- if (before == after) {
- return;
- }
- if (compareAndSet(before, after)) {
- if (isClosed(after) && _parent == null) {
- onClosed(dataSum(after));
- }
- return;
- }
- }
- }
-
- /** If possible returns the <code>dataSum</code> that would be delivered
- * to {@link #onClosed(int)} if this epoch were closed at this moment,
- * otherwise returns null. This will succeed if and only if the tree
- * consists only of a single node.
- */
- public Integer attemptDataSum() {
- final long state = get();
- if (!isAnyChildPresent(state) && entryCount(state) == 0) {
- // this is better than Integer.valueOf for dynamic escape analysis
- //return new Integer(dataSum(state));
- // this is better than new Integer() for object creation
- return Integer.valueOf(dataSum(state));
- }
- else {
- return null;
- }
- }
-
- /** For debugging purposes. */
- int computeSpread() {
- final long state = get();
- if (isAnyChildPresent(state)) {
- int sum = 0;
- for (int which = 0; which < BF; ++which) {
- final EpochNode child = getChild(state, which);
- if (child != null) {
- sum += child.computeSpread();
- }
- else {
- // child would be created for arrive, so count it
- sum += 1;
- }
- }
- return sum;
- }
- else {
- return 1;
- }
- }
-}
\ No newline at end of file
[25/50] [abbrv] ignite git commit: IGNITE-9054 Avoid using
OptimizedMarshaller with initial ScanQuery. - Fixes #4592.
Posted by ag...@apache.org.
IGNITE-9054 Avoid using OptimizedMarshaller with initial ScanQuery. - Fixes #4592.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9bfc823e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9bfc823e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9bfc823e
Branch: refs/heads/ignite-5960
Commit: 9bfc823e012b052565a32ab1c82118fe0aff950f
Parents: f4c18f1
Author: Ilya Kasnacheev <il...@gmail.com>
Authored: Tue Sep 4 17:26:58 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Sep 4 18:17:27 2018 +0300
----------------------------------------------------------------------
.../cache/query/GridCacheQueryManager.java | 4 +-
.../query/GridCacheQueryResponseEntry.java | 5 +-
.../ignite/custom/DummyEventFilterFactory.java | 10 +-
.../ContinuousQueryMarshallerTest.java | 168 +++++++++++++++++++
.../ContinuousQueryPeerClassLoadingTest.java | 4 +-
.../IgniteCacheQuerySelfTestSuite3.java | 2 +
6 files changed, 183 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bfc823e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 281400e..982006f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -1278,7 +1278,7 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
continue;
}
else
- data.add(!loc ? new GridCacheQueryResponseEntry<>(key, val) : F.t(key, val));
+ data.add(new T2<>(key, val));
}
if (!loc) {
@@ -3119,7 +3119,7 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
}
}
else
- next0 = !locNode ? new GridCacheQueryResponseEntry<>(key0, val0):
+ next0 = !locNode ? new T2<>(key0, val0):
new CacheQueryEntry<>(key0, val0);
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bfc823e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponseEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponseEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponseEntry.java
index 2c1a4f5..650f0c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponseEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponseEntry.java
@@ -27,7 +27,10 @@ import org.apache.ignite.internal.util.tostring.GridToStringInclude;
/**
* Class to store query results returned by remote nodes. It's required to fully
* control serialization process. Local entries can be returned to user as is.
+ * <p>
+ * @deprecated Should be removed in Apache Ignite 3.0.
*/
+@Deprecated
public class GridCacheQueryResponseEntry<K, V> implements Map.Entry<K, V>, Externalizable {
/** */
private static final long serialVersionUID = 0L;
@@ -113,4 +116,4 @@ public class GridCacheQueryResponseEntry<K, V> implements Map.Entry<K, V>, Exter
@Override public String toString() {
return "[" + key + "=" + val + "]";
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bfc823e/modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java b/modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java
index e0688bc..103e6a8 100644
--- a/modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java
+++ b/modules/core/src/test/java/org/apache/ignite/custom/DummyEventFilterFactory.java
@@ -25,22 +25,22 @@ import javax.cache.event.CacheEntryListenerException;
/**
* Must be not in org.apache.ignite.internal
*/
-public class DummyEventFilterFactory implements Factory<CacheEntryEventFilter<Integer, String>> {
+public class DummyEventFilterFactory<T> implements Factory<CacheEntryEventFilter<Integer, T>> {
/** */
private static final long serialVersionUID = 0L;
/** {@inheritDoc} */
- @Override public CacheEntryEventFilter<Integer, String> create() {
- return new DummyEventFilter();
+ @Override public CacheEntryEventFilter<Integer, T> create() {
+ return new DummyEventFilter<T>();
}
/**
*
*/
- private static class DummyEventFilter implements CacheEntryEventFilter<Integer, String> {
+ private static class DummyEventFilter<T> implements CacheEntryEventFilter<Integer, T> {
/** {@inheritDoc} */
@Override public boolean evaluate(
- final CacheEntryEvent<? extends Integer, ? extends String> evt) throws CacheEntryListenerException {
+ final CacheEntryEvent<? extends Integer, ? extends T> evt) throws CacheEntryListenerException {
return true;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bfc823e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryMarshallerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryMarshallerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryMarshallerTest.java
new file mode 100644
index 0000000..44dcc1c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryMarshallerTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.ObjectStreamException;
+import java.io.Serializable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import javax.cache.Cache;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryListenerException;
+import javax.cache.event.CacheEntryUpdatedListener;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.custom.DummyEventFilterFactory;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Checks that Optimized Marshaller is not used on any stage of Continuous Query handling.
+ */
+public class ContinuousQueryMarshallerTest extends GridCommonAbstractTest {
+ /** */
+ public static final String CACHE_NAME = "test-cache";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+ final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setClientMode(gridName.contains("client"));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRemoteFilterFactoryClient() throws Exception {
+ check("server", "client");
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRemoteFilterFactoryServer() throws Exception {
+ check("server1", "server2");
+ }
+
+ /**
+ * @param node1Name Node 1 name.
+ * @param node2Name Node 2 name.
+ */
+ private void check(String node1Name, String node2Name) throws Exception {
+ final Ignite node1 = startGrid(node1Name);
+
+ final IgniteCache<Integer, MarshallerCheckingEntry> cache = node1.getOrCreateCache(CACHE_NAME);
+
+ for (int i = 0; i < 10; i++)
+ cache.put(i, new MarshallerCheckingEntry(String.valueOf(i)));
+
+ final Ignite node2 = startGrid(node2Name);
+
+ final ContinuousQuery<Integer, MarshallerCheckingEntry> qry = new ContinuousQuery<>();
+
+ ScanQuery<Integer, MarshallerCheckingEntry> scanQry = new ScanQuery<>(new IgniteBiPredicate<Integer, MarshallerCheckingEntry>() {
+ @Override public boolean apply(Integer key, MarshallerCheckingEntry val) {
+ return key % 2 == 0;
+ }
+ });
+
+ qry.setInitialQuery(scanQry);
+
+ qry.setRemoteFilterFactory(new DummyEventFilterFactory<>());
+
+ final CountDownLatch latch = new CountDownLatch(15);
+
+ qry.setLocalListener(new CacheEntryUpdatedListener<Integer, MarshallerCheckingEntry>() {
+ @Override public void onUpdated(
+ final Iterable<CacheEntryEvent<? extends Integer, ? extends MarshallerCheckingEntry>> evts)
+ throws CacheEntryListenerException {
+
+ System.out.println(">> Client 1 events " + evts);
+
+ for (CacheEntryEvent<? extends Integer, ? extends MarshallerCheckingEntry> evt : evts)
+ latch.countDown();
+ }
+ });
+
+ final IgniteCache<Integer, MarshallerCheckingEntry> cache1 = node2.cache(CACHE_NAME);
+
+ for (Cache.Entry<Integer, MarshallerCheckingEntry> entry : cache1.query(qry)) {
+ latch.countDown();
+ System.out.println(">> Initial entry " + entry);
+ }
+
+ for (int i = 10; i < 20; i++)
+ cache1.put(i, new MarshallerCheckingEntry(i));
+
+ assertTrue(Long.toString(latch.getCount()), latch.await(5, TimeUnit.SECONDS));
+ }
+
+ /** Checks that OptimizedMarshaller is never used (BinaryMarshaller is OK) */
+ private class MarshallerCheckingEntry implements Serializable, Binarylizable {
+ /** */
+ private Object val;
+
+ /** */
+ public MarshallerCheckingEntry(Object val) {
+ this.val = val;
+ }
+
+ /** */
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** */
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** */
+ private void readObjectNoData() throws ObjectStreamException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** */
+ @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+ writer.writeObject("value", val);
+ }
+
+ /** */
+ @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+ val = reader.readObject("value");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bfc823e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java
index e5d1d60..73d8d0d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryPeerClassLoadingTest.java
@@ -91,8 +91,8 @@ public class ContinuousQueryPeerClassLoadingTest extends GridCommonAbstractTest
final ContinuousQuery<Integer, String> qry1 = new ContinuousQuery<>();
final ContinuousQuery<Integer, String> qry2 = new ContinuousQuery<>();
- qry1.setRemoteFilterFactory(new DummyEventFilterFactory());
- qry2.setRemoteFilterFactory(new DummyEventFilterFactory());
+ qry1.setRemoteFilterFactory(new DummyEventFilterFactory<>());
+ qry2.setRemoteFilterFactory(new DummyEventFilterFactory<>());
final AtomicInteger client1Evts = new AtomicInteger(0);
final AtomicInteger client2Evts = new AtomicInteger(0);
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bfc823e/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
index e810d30..08511d9 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite3.java
@@ -47,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBin
import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationTest;
import org.apache.ignite.internal.processors.cache.query.continuous.CacheKeepBinaryIterationStoreEnabledTest;
import org.apache.ignite.internal.processors.cache.query.continuous.ClientReconnectContinuousQueryTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryMarshallerTest;
import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryPeerClassLoadingTest;
import org.apache.ignite.internal.processors.cache.query.continuous.ContinuousQueryRemoteFilterMissingInClassPathSelfTest;
import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicNearEnabledSelfTest;
@@ -132,6 +133,7 @@ public class IgniteCacheQuerySelfTestSuite3 extends TestSuite {
suite.addTestSuite(ClientReconnectContinuousQueryTest.class);
suite.addTestSuite(ContinuousQueryPeerClassLoadingTest.class);
suite.addTestSuite(ClientReconnectContinuousQueryTest.class);
+ suite.addTestSuite(ContinuousQueryMarshallerTest.class);
suite.addTestSuite(CacheContinuousQueryConcurrentPartitionUpdateTest.class);
suite.addTestSuite(CacheContinuousQueryEventBufferTest.class);
[06/50] [abbrv] ignite git commit: IGNITE-9273 Log logical records to
WAL for LOCAL caches
Posted by ag...@apache.org.
IGNITE-9273 Log logical records to WAL for LOCAL caches
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7e9fd4e1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7e9fd4e1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7e9fd4e1
Branch: refs/heads/ignite-5960
Commit: 7e9fd4e122c00c0968b278f3b40475b6a737c6ad
Parents: 2cccfae
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Fri Aug 31 17:55:17 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Aug 31 17:55:17 2018 +0300
----------------------------------------------------------------------
.../processors/cache/GridCacheMapEntry.java | 4 +
.../cache/IgniteCacheOffheapManagerImpl.java | 2 +-
.../GridCacheDatabaseSharedManager.java | 2 +-
.../db/wal/IgniteWalRecoveryTest.java | 1906 -----------------
.../IgniteWalRecoveryWithCompactionTest.java | 33 -
.../db/wal/IgniteWalRecoveryTest.java | 1911 ++++++++++++++++++
.../IgniteWalRecoveryWithCompactionTest.java | 33 +
7 files changed, 1950 insertions(+), 1941 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e9fd4e1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 249fbb3..4dc4944 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -2037,6 +2037,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
update(updated, expireTime, ttl, ver, true);
+ logUpdate(op, updated, ver, expireTime, 0);
+
if (evt) {
CacheObject evtOld = null;
@@ -2067,6 +2069,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
update(null, CU.TTL_ETERNAL, CU.EXPIRE_TIME_ETERNAL, ver, true);
+ logUpdate(op, null, ver, CU.EXPIRE_TIME_ETERNAL, 0);
+
if (evt) {
CacheObject evtOld = null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e9fd4e1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 91c6f3e..bc35264 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -295,7 +295,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
* @param part Partition.
* @return Data store for given entry.
*/
- public CacheDataStore dataStore(GridDhtLocalPartition part) {
+ @Override public CacheDataStore dataStore(GridDhtLocalPartition part) {
if (grp.isLocal())
return locCacheDataStore;
else {
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e9fd4e1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 7393c01..2e5f26f 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -2492,7 +2492,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
if (partId == -1)
partId = cacheCtx.affinity().partition(dataEntry.key());
- GridDhtLocalPartition locPart = cacheCtx.topology().forceCreatePartition(partId);
+ GridDhtLocalPartition locPart = cacheCtx.isLocal() ? null : cacheCtx.topology().forceCreatePartition(partId);
switch (dataEntry.op()) {
case CREATE:
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e9fd4e1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
deleted file mode 100644
index 116f174..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
+++ /dev/null
@@ -1,1906 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence.db.wal;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Random;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.TimeUnit;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteCompute;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.query.SqlFieldsQuery;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.BinaryConfiguration;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.wal.WALIterator;
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
-import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
-import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
-import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
-import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor;
-import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
-import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.internal.util.GridUnsafe;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.PA;
-import org.apache.ignite.internal.util.typedef.PAX;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.lang.IgniteCallable;
-import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.lang.IgniteRunnable;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy;
-import org.apache.ignite.transactions.Transaction;
-import org.apache.ignite.transactions.TransactionConcurrency;
-import org.apache.ignite.transactions.TransactionIsolation;
-import org.junit.Assert;
-
-import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
-
-/**
- *
- */
-public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
- /** */
- private static final String HAS_CACHE = "HAS_CACHE";
-
- /** */
- private static final int LARGE_ARR_SIZE = 1025;
-
- /** */
- private boolean fork;
-
- /** */
- private String cacheName;
-
- /** */
- private int walSegmentSize;
-
- /** Log only. */
- private boolean logOnly;
-
- /** {@inheritDoc} */
- @Override protected boolean isMultiJvm() {
- return fork;
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(cacheName);
-
- ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
- ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
- ccfg.setNodeFilter(new RemoteNodeFilter());
- ccfg.setIndexedTypes(Integer.class, IndexedObject.class);
-
- cfg.setCacheConfiguration(ccfg);
-
- DataStorageConfiguration dbCfg = new DataStorageConfiguration();
-
- dbCfg.setPageSize(4 * 1024);
-
- DataRegionConfiguration memPlcCfg = new DataRegionConfiguration();
-
- memPlcCfg.setName("dfltDataRegion");
- memPlcCfg.setInitialSize(1024L * 1024 * 1024);
- memPlcCfg.setMaxSize(1024L * 1024 * 1024);
- memPlcCfg.setPersistenceEnabled(true);
-
- dbCfg.setDefaultDataRegionConfiguration(memPlcCfg);
-
- dbCfg.setWalRecordIteratorBufferSize(1024 * 1024);
-
- dbCfg.setWalHistorySize(2);
-
- if (logOnly)
- dbCfg.setWalMode(WALMode.LOG_ONLY);
-
- if (walSegmentSize != 0)
- dbCfg.setWalSegmentSize(walSegmentSize);
-
- cfg.setDataStorageConfiguration(dbCfg);
-
- cfg.setMarshaller(null);
-
- BinaryConfiguration binCfg = new BinaryConfiguration();
-
- binCfg.setCompactFooter(false);
-
- cfg.setBinaryConfiguration(binCfg);
-
- if (!getTestIgniteInstanceName(0).equals(gridName))
- cfg.setUserAttributes(F.asMap(HAS_CACHE, true));
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- stopAllGrids();
-
- cleanPersistenceDir();
-
- cacheName = "partitioned";
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- logOnly = false;
-
- cleanPersistenceDir();
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalBig() throws Exception {
- IgniteEx ignite = startGrid(1);
-
- ignite.active(true);
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- Random rnd = new Random();
-
- Map<Integer, IndexedObject> map = new HashMap<>();
-
- for (int i = 0; i < 10_000; i++) {
- if (i % 1000 == 0)
- X.println(" >> " + i);
-
- int k = rnd.nextInt(300_000);
- IndexedObject v = new IndexedObject(rnd.nextInt(10_000));
-
- cache.put(k, v);
- map.put(k, v);
- }
-
- // Check.
- for (Integer k : map.keySet())
- assertEquals(map.get(k), cache.get(k));
-
- stopGrid(1);
-
- ignite = startGrid(1);
-
- ignite.active(true);
-
- cache = ignite.cache("partitioned");
-
- // Check.
- for (Integer k : map.keySet())
- assertEquals(map.get(k), cache.get(k));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalBigObjectNodeCancel() throws Exception {
- final int MAX_SIZE_POWER = 21;
-
- IgniteEx ignite = startGrid(1);
-
- ignite.active(true);
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < MAX_SIZE_POWER; ++i) {
- int size = 1 << i;
-
- cache.put("key_" + i, createTestData(size));
- }
-
- stopGrid(1, true);
-
- ignite = startGrid(1);
-
- ignite.active(true);
-
- cache = ignite.cache("partitioned");
-
- // Check.
- for (int i = 0; i < MAX_SIZE_POWER; ++i) {
- int size = 1 << i;
-
- int[] data = createTestData(size);
-
- int[] val = (int[])cache.get("key_" + i);
-
- assertTrue("Invalid data. [key=key_" + i + ']', Arrays.equals(data, val));
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testSwitchClassLoader() throws Exception {
- try {
- final IgniteEx igniteEx = startGrid(1);
-
- // CustomDiscoveryMessage will trigger service tasks
- startGrid(2);
-
- igniteEx.active(true);
-
- IgniteCache<Integer, EnumVal> cache = igniteEx.cache("partitioned");
-
- // Creates LoadCacheJobV2
-// cache.loadCache(null);
-
- final ClassLoader oldCl = Thread.currentThread().getContextClassLoader();
- final ClassLoader newCl = getExternalClassLoader();
-
- Thread.currentThread().setContextClassLoader(newCl);
-
- for (int i = 0; i < 10; i++)
- cache.put(i, i % 2 == 0 ? EnumVal.VAL1 : EnumVal.VAL2);
-
- for (int i = 0; i < 10; i++)
- assert cache.containsKey(i);
-
- // Invokes ClearTask with new class loader
- cache.clear();
-
- Thread.currentThread().setContextClassLoader(oldCl);
-
- for (int i = 0; i < 10; i++)
- cache.put(i, i % 2 == 0 ? EnumVal.VAL1 : EnumVal.VAL2);
-
- for (int i = 0; i < 10; i++)
- assert cache.containsKey(i);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalSimple() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- ignite.active(true);
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- info(" --> step1");
-
- for (int i = 0; i < 10_000; i += 2) {
-// X.println(" -> put: " + i);
-
- cache.put(i, new IndexedObject(i));
- }
-
- info(" --> step2");
-
- for (int i = 0; i < 10_000; i += 3)
- cache.put(i, new IndexedObject(i * 2));
-
- info(" --> step3");
-
- for (int i = 0; i < 10_000; i += 7)
- cache.put(i, new IndexedObject(i * 3));
-
- info(" --> check1");
-
- // Check.
- for (int i = 0; i < 10_000; i++) {
- IndexedObject o;
-
- if (i % 7 == 0)
- o = new IndexedObject(i * 3);
- else if (i % 3 == 0)
- o = new IndexedObject(i * 2);
- else if (i % 2 == 0)
- o = new IndexedObject(i);
- else
- o = null;
-
- assertEquals(o, cache.get(i));
- }
-
- stopGrid(1);
-
- ignite = startGrid(1);
-
- ignite.active(true);
-
- cache = ignite.cache("partitioned");
-
- info(" --> check2");
-
- // Check.
- for (int i = 0; i < 10_000; i++) {
- IndexedObject o;
-
- if (i % 7 == 0)
- o = new IndexedObject(i * 3);
- else if (i % 3 == 0)
- o = new IndexedObject(i * 2);
- else if (i % 2 == 0)
- o = new IndexedObject(i);
- else
- o = null;
-
- assertEquals(o, cache.get(i));
- }
-
- info(" --> ok");
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testWalLargeValue() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- ignite.active(true);
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < 10_000; i++) {
- final byte[] data = new byte[i];
-
- Arrays.fill(data, (byte)i);
-
- cache.put(i, data);
-
- if (i % 1000 == 0)
- X.println(" ---> put: " + i);
-
-// Assert.assertArrayEquals(data, (byte[])cache.get(i));
- }
-
-// info(" --> check1");
-//
-// for (int i = 0; i < 25_000; i++) {
-// final byte[] data = new byte[i];
-//
-// Arrays.fill(data, (byte)i);
-//
-// final byte[] loaded = (byte[]) cache.get(i);
-//
-// Assert.assertArrayEquals(data, loaded);
-// }
-
- stopGrid(1);
-
- ignite = startGrid(1);
-
- ignite.active(true);
-
- cache = ignite.cache("partitioned");
-
- info(" --> check2");
-
- for (int i = 0; i < 10_000; i++) {
- final byte[] data = new byte[i];
-
- Arrays.fill(data, (byte)i);
-
- final byte[] loaded = (byte[]) cache.get(i);
-
- Assert.assertArrayEquals(data, loaded);
-
- if (i % 1000 == 0)
- X.println(" ---> get: " + i);
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalRolloverMultithreadedDefault() throws Exception {
- logOnly = false;
-
- checkWalRolloverMultithreaded();
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalRolloverMultithreadedLogOnly() throws Exception {
- logOnly = true;
-
- checkWalRolloverMultithreaded();
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testHugeCheckpointRecord() throws Exception {
- try {
- final IgniteEx ignite = startGrid(1);
-
- ignite.active(true);
-
- for (int i = 0; i < 50; i++) {
- CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>("cache-" + i);
-
- // We can get 'too many open files' with default number of partitions.
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 128));
-
- IgniteCache<Object, Object> cache = ignite.getOrCreateCache(ccfg);
-
- cache.put(i, i);
- }
-
- final long endTime = System.currentTimeMillis() + 30_000;
-
- IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- Random rnd = ThreadLocalRandom.current();
-
- while (U.currentTimeMillis() < endTime) {
- IgniteCache<Object, Object> cache = ignite.cache("cache-" + rnd.nextInt(50));
-
- cache.put(rnd.nextInt(50_000), rnd.nextInt());
- }
-
- return null;
- }
- }, 16, "put-thread");
-
- while (System.currentTimeMillis() < endTime) {
- ignite.context().cache().context().database().wakeupForCheckpoint("test").get();
-
- U.sleep(500);
- }
-
- fut.get();
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- private void checkWalRolloverMultithreaded() throws Exception {
- walSegmentSize = 2 * 1024 * 1024;
-
- final long endTime = System.currentTimeMillis() + 2 * 60 * 1000;
-
- try {
- IgniteEx ignite = startGrid(1);
-
- ignite.active(true);
-
- final IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- GridTestUtils.runMultiThreaded(new Callable<Void>() {
- @Override public Void call() throws Exception {
- Random rnd = ThreadLocalRandom.current();
-
- while (U.currentTimeMillis() < endTime)
- cache.put(rnd.nextInt(50_000), rnd.nextInt());
-
- return null;
- }
- }, 16, "put-thread");
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testWalRenameDirSimple() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- ignite.active(true);
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < 100; i++)
- cache.put(i, new IndexedObject(i));
-
- final Object consistentId = ignite.cluster().localNode().consistentId();
-
- stopGrid(1);
-
- final File cacheDir = cacheDir("partitioned", consistentId.toString());
-
- final boolean renamed = cacheDir.renameTo(new File(cacheDir.getParent(), "cache-partitioned0"));
-
- assert renamed;
-
- cacheName = "partitioned0";
-
- ignite = startGrid(1);
-
- ignite.active(true);
-
- cache = ignite.cache(cacheName);
-
- for (int i = 0; i < 100; i++)
- assertEquals(new IndexedObject(i), cache.get(i));
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @param cacheName Cache name.
- * @param consId Consistent ID.
- * @return Cache dir.
- * @throws IgniteCheckedException If fail.
- */
- private File cacheDir(final String cacheName, final String consId) throws IgniteCheckedException {
- final String subfolderName
- = PdsConsistentIdProcessor.genNewStyleSubfolderName(0, UUID.fromString(consId));
-
- final File dbDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false);
-
- assert dbDir.exists();
-
- final File consIdDir = new File(dbDir.getAbsolutePath(), subfolderName);
-
- assert consIdDir.exists();
-
- final File cacheDir = new File(consIdDir.getAbsolutePath(), "cache-" + cacheName);
-
- assert cacheDir.exists();
-
- return cacheDir;
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRecoveryNoCheckpoint() throws Exception {
- try {
- IgniteEx ctrlGrid = startGrid(0);
-
- fork = true;
-
- IgniteEx cacheGrid = startGrid(1);
-
- ctrlGrid.active(true);
-
- ctrlGrid.compute(ctrlGrid.cluster().forRemotes()).run(new LoadRunnable(false));
-
- info("Killing remote process...");
-
- ((IgniteProcessProxy)cacheGrid).kill();
-
- final IgniteEx g0 = ctrlGrid;
-
- GridTestUtils.waitForCondition(new PA() {
- /** {@inheritDoc} */
- @Override public boolean apply() {
- return g0.cluster().nodes().size() == 1;
- }
- }, getTestTimeout());
-
- fork = false;
-
- // Now start the grid and verify that updates were restored from WAL.
- cacheGrid = startGrid(1);
-
- IgniteCache<Object, Object> cache = cacheGrid.cache("partitioned");
-
- for (int i = 0; i < 10_000; i++)
- assertEquals(new IndexedObject(i), cache.get(i));
-
- List<List<?>> res = cache.query(new SqlFieldsQuery("select count(iVal) from IndexedObject")).getAll();
-
- assertEquals(1, res.size());
- assertEquals(10_000L, res.get(0).get(0));
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRecoveryLargeNoCheckpoint() throws Exception {
- try {
- IgniteEx ctrlGrid = startGrid(0);
-
- fork = true;
-
- IgniteEx cacheGrid = startGrid(1);
-
- ctrlGrid.active(true);
-
- ctrlGrid.compute(ctrlGrid.cluster().forRemotes()).run(new LargeLoadRunnable(false));
-
- info("Killing remote process...");
-
- ((IgniteProcessProxy)cacheGrid).kill();
-
- final IgniteEx g0 = ctrlGrid;
-
- GridTestUtils.waitForCondition(new PA() {
- /** {@inheritDoc} */
- @Override public boolean apply() {
- return g0.cluster().nodes().size() == 1;
- }
- }, getTestTimeout());
-
- fork = false;
-
- // Now start the grid and verify that updates were restored from WAL.
- cacheGrid = startGrid(1);
-
- IgniteCache<Object, Object> cache = cacheGrid.cache("partitioned");
-
- for (int i = 0; i < 1000; i++) {
- final long[] data = new long[LARGE_ARR_SIZE];
-
- Arrays.fill(data, i);
-
- final long[] loaded = (long[]) cache.get(i);
-
- Assert.assertArrayEquals(data, loaded);
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return TimeUnit.MINUTES.toMillis(20);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRandomCrash() throws Exception {
- try {
- IgniteEx ctrlGrid = startGrid(0);
-
- fork = true;
-
- IgniteEx cacheGrid = startGrid(1);
-
- ctrlGrid.active(true);
-
- IgniteCompute rmt = ctrlGrid.compute(ctrlGrid.cluster().forRemotes());
-
- rmt.run(new LoadRunnable(false));
-
- info(">>> Finished cache population.");
-
- rmt.run(new AsyncLoadRunnable());
-
- Thread.sleep(20_000);
-
- info(">>> Killing remote process...");
-
- ((IgniteProcessProxy)cacheGrid).kill();
-
- startGrid(1);
-
- Boolean res = rmt.call(new VerifyCallable());
-
- assertTrue(res);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testLargeRandomCrash() throws Exception {
- try {
- IgniteEx ctrlGrid = startGrid(0);
-
- fork = true;
-
- IgniteEx cacheGrid = startGrid(1);
-
- ctrlGrid.active(true);
-
- IgniteCompute rmt = ctrlGrid.compute(ctrlGrid.cluster().forRemotes());
-
- rmt.run(new LargeLoadRunnable(false));
-
- info(">>> Finished cache population.");
-
- rmt.run(new AsyncLargeLoadRunnable());
-
- Thread.sleep(20_000);
-
- info(">>> Killing remote process...");
-
- ((IgniteProcessProxy)cacheGrid).kill();
-
- startGrid(1);
-
- Boolean res = rmt.call(new VerifyLargeCallable());
-
- assertTrue(res);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- *
- */
- private static class RemoteNodeFilter implements IgnitePredicate<ClusterNode> {
- /** {@inheritDoc} */
- @Override public boolean apply(ClusterNode clusterNode) {
- return clusterNode.attribute(HAS_CACHE) != null;
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testDestroyCache() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- ignite.active(true);
-
- IgniteCache<Object, Object> cache = ignite.getOrCreateCache("test");
-
- cache.put(1, new IndexedObject(1));
-
- ignite.destroyCache("test");
-
- cache = ignite.getOrCreateCache("test");
-
- // No entry available after cache destroy.
- assertNull(cache.get(1));
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testEvictPartition() throws Exception {
- try {
- Ignite ignite1 = startGrid("node1");
-
- ignite1.active(true);
-
- IgniteCache<Object, Object> cache1 = ignite1.cache(cacheName);
-
- for (int i = 0; i < 100; i++)
- cache1.put(i, new IndexedObject(i));
-
- Ignite ignite2 = startGrid("node2");
-
- IgniteCache<Object, Object> cache2 = ignite2.cache(cacheName);
-
- for (int i = 0; i < 100; i++) {
- assertEquals(new IndexedObject(i), cache1.get(i));
- assertEquals(new IndexedObject(i), cache2.get(i));
- }
-
- ignite1.close();
- ignite2.close();
-
- ignite1 = startGrid("node1");
- ignite2 = startGrid("node2");
-
- ignite1.active(true);
-
- cache1 = ignite1.cache(cacheName);
- cache2 = ignite2.cache(cacheName);
-
- for (int i = 0; i < 100; i++) {
- assertEquals(new IndexedObject(i), cache1.get(i));
- assertEquals(new IndexedObject(i), cache2.get(i));
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testMetastorage() throws Exception {
- try {
- int cnt = 5000;
-
- IgniteEx ignite0 = (IgniteEx)startGrid("node1");
- IgniteEx ignite1 = (IgniteEx)startGrid("node2");
-
- ignite1.active(true);
-
- GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
- GridCacheSharedContext<Object, Object> sharedCtx1 = ignite1.context().cache().context();
-
- MetaStorage storage0 = sharedCtx0.database().metaStorage();
- MetaStorage storage1 = sharedCtx1.database().metaStorage();
-
- assert storage0 != null;
-
- for (int i = 0; i < cnt; i++) {
- sharedCtx0.database().checkpointReadLock();
-
- try {
- storage0.putData(String.valueOf(i), new byte[]{(byte)(i % 256), 2, 3});
- }
- finally {
- sharedCtx0.database().checkpointReadUnlock();
- }
-
- byte[] b1 = new byte[i + 3];
- b1[0] = 1;
- b1[1] = 2;
- b1[2] = 3;
-
- sharedCtx1.database().checkpointReadLock();
-
- try {
- storage1.putData(String.valueOf(i), b1);
- }
- finally {
- sharedCtx1.database().checkpointReadUnlock();
- }
- }
-
- for (int i = 0; i < cnt; i++) {
- byte[] d1 = storage0.getData(String.valueOf(i));
- assertEquals(3, d1.length);
- assertEquals((byte)(i % 256), d1[0]);
- assertEquals(2, d1[1]);
- assertEquals(3, d1[2]);
-
- byte[] d2 = storage1.getData(String.valueOf(i));
- assertEquals(i + 3, d2.length);
- assertEquals(1, d2[0]);
- assertEquals(2, d2[1]);
- assertEquals(3, d2[2]);
- }
-
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testMetastorageLargeArray() throws Exception {
- try {
- int cnt = 5000;
- int arraySize = 32_768;
-
- IgniteEx ignite = (IgniteEx)startGrid("node1");
-
- ignite.active(true);
-
- GridCacheSharedContext<Object, Object> sharedCtx = ignite.context().cache().context();
-
- MetaStorage storage = sharedCtx.database().metaStorage();
-
- for (int i = 0; i < cnt; i++) {
- byte[] b1 = new byte[arraySize];
- for (int k = 0; k < arraySize; k++) {
- b1[k] = (byte) (k % 100);
- }
-
- sharedCtx.database().checkpointReadLock();
-
- try {
- storage.putData(String.valueOf(i), b1);
- }
- finally {
- sharedCtx.database().checkpointReadUnlock();
- }
- }
-
- for (int i = 0; i < cnt; i++) {
- byte[] d2 = storage.getData(String.valueOf(i));
- assertEquals(arraySize, d2.length);
-
- for (int k = 0; k < arraySize; k++) {
- assertEquals((byte) (k % 100), d2[k]);
- }
- }
-
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testMetastorageRemove() throws Exception {
- try {
- int cnt = 400;
-
- IgniteEx ignite0 = (IgniteEx)startGrid("node1");
-
- ignite0.active(true);
-
- GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
-
- MetaStorage storage = sharedCtx0.database().metaStorage();
-
- assert storage != null;
-
- for (int i = 0; i < cnt; i++) {
- sharedCtx0.database().checkpointReadLock();
-
- try {
- storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
- }
- finally {
- sharedCtx0.database().checkpointReadUnlock();
- }
- }
-
- for (int i = 0; i < 10; i++) {
- sharedCtx0.database().checkpointReadLock();
-
- try {
- storage.removeData(String.valueOf(i));
- }
- finally {
- sharedCtx0.database().checkpointReadUnlock();
- }
- }
-
- for (int i = 10; i < cnt; i++) {
- byte[] d1 = storage.getData(String.valueOf(i));
- assertEquals(3, d1.length);
- assertEquals(1, d1[0]);
- assertEquals(2, d1[1]);
- assertEquals(3, d1[2]);
- }
-
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testMetastorageUpdate() throws Exception {
- try {
- int cnt = 2000;
-
- IgniteEx ignite0 = (IgniteEx)startGrid("node1");
-
- ignite0.active(true);
-
- GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
-
- MetaStorage storage = sharedCtx0.database().metaStorage();
-
- assert storage != null;
-
- for (int i = 0; i < cnt; i++) {
- sharedCtx0.database().checkpointReadLock();
-
- try {
- storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
- }
- finally {
- sharedCtx0.database().checkpointReadUnlock();
- }
- }
-
- for (int i = 0; i < cnt; i++) {
- sharedCtx0.database().checkpointReadLock();
-
- try {
- storage.putData(String.valueOf(i), new byte[]{2, 2, 3, 4});
- }
- finally {
- sharedCtx0.database().checkpointReadUnlock();
- }
- }
-
- for (int i = 0; i < cnt; i++) {
- byte[] d1 = storage.getData(String.valueOf(i));
- assertEquals(4, d1.length);
- assertEquals(2, d1[0]);
- assertEquals(2, d1[1]);
- assertEquals(3, d1[2]);
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testMetastorageWalRestore() throws Exception {
- try {
- int cnt = 2000;
-
- IgniteEx ignite0 = (IgniteEx)startGrid(0);
-
- ignite0.active(true);
-
- GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
-
- MetaStorage storage = sharedCtx0.database().metaStorage();
-
- assert storage != null;
-
- for (int i = 0; i < cnt; i++) {
- sharedCtx0.database().checkpointReadLock();
-
- try {
- storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
- }
- finally {
- sharedCtx0.database().checkpointReadUnlock();
- }
- }
-
- for (int i = 0; i < cnt; i++) {
- byte[] value = storage.getData(String.valueOf(i));
- assert value != null;
- assert value.length == 3;
- }
-
- stopGrid(0);
-
- ignite0 = startGrid(0);
-
- ignite0.active(true);
-
- sharedCtx0 = ignite0.context().cache().context();
-
- storage = sharedCtx0.database().metaStorage();
-
- assert storage != null;
-
- for (int i = 0; i < cnt; i++) {
- byte[] value = storage.getData(String.valueOf(i));
- assert value != null;
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testApplyDeltaRecords() throws Exception {
- try {
- IgniteEx ignite0 = (IgniteEx)startGrid("node0");
-
- ignite0.active(true);
-
- IgniteCache<Object, Object> cache0 = ignite0.cache(cacheName);
-
- for (int i = 0; i < 1000; i++)
- cache0.put(i, new IndexedObject(i));
-
- GridCacheSharedContext<Object, Object> sharedCtx = ignite0.context().cache().context();
-
- GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)sharedCtx.database();
-
- db.waitForCheckpoint("test");
- db.enableCheckpoints(false).get();
-
- // Log something to know where to start.
- WALPointer ptr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
-
- info("Replay marker: " + ptr);
-
- for (int i = 1000; i < 5000; i++)
- cache0.put(i, new IndexedObject(i));
-
- info("Done puts...");
-
- for (int i = 2_000; i < 3_000; i++)
- cache0.remove(i);
-
- info("Done removes...");
-
- for (int i = 5000; i < 6000; i++)
- cache0.put(i, new IndexedObject(i));
-
- info("Done puts...");
-
- Map<FullPageId, byte[]> rolledPages = new HashMap<>();
-
- int pageSize = sharedCtx.database().pageSize();
-
- ByteBuffer buf = ByteBuffer.allocateDirect(pageSize);
-
- // Now check that deltas can be correctly applied.
- try (WALIterator it = sharedCtx.wal().replay(ptr)) {
- while (it.hasNext()) {
- IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
-
- WALRecord rec = tup.get2();
-
- if (rec instanceof PageSnapshot) {
- PageSnapshot page = (PageSnapshot)rec;
-
- rolledPages.put(page.fullPageId(), page.pageData());
- }
- else if (rec instanceof PageDeltaRecord) {
- PageDeltaRecord delta = (PageDeltaRecord)rec;
-
- FullPageId fullId = new FullPageId(delta.pageId(), delta.groupId());
-
- byte[] pageData = rolledPages.get(fullId);
-
- if (pageData == null) {
- pageData = new byte[pageSize];
-
- rolledPages.put(fullId, pageData);
- }
-
- assertNotNull("Missing page snapshot [page=" + fullId + ", delta=" + delta + ']', pageData);
-
- buf.order(ByteOrder.nativeOrder());
-
- buf.position(0);
- buf.put(pageData);
- buf.position(0);
-
- delta.applyDelta(sharedCtx.database().dataRegion(null).pageMemory(),
- GridUnsafe.bufferAddress(buf));
-
- buf.position(0);
-
- buf.get(pageData);
- }
- }
- }
-
- info("Done apply...");
-
- PageMemoryEx pageMem = (PageMemoryEx)db.dataRegion(null).pageMemory();
-
- for (Map.Entry<FullPageId, byte[]> entry : rolledPages.entrySet()) {
- FullPageId fullId = entry.getKey();
-
- ignite0.context().cache().context().database().checkpointReadLock();
-
- try {
- long page = pageMem.acquirePage(fullId.groupId(), fullId.pageId(), true);
-
- try {
- long bufPtr = pageMem.writeLock(fullId.groupId(), fullId.pageId(), page, true);
-
- try {
- byte[] data = entry.getValue();
-
- for (int i = 0; i < data.length; i++) {
- if (fullId.pageId() == TrackingPageIO.VERSIONS.latest().trackingPageFor(fullId.pageId(), db.pageSize()))
- continue; // Skip tracking pages.
-
- assertEquals("page=" + fullId + ", pos=" + i, PageUtils.getByte(bufPtr, i), data[i]);
- }
- }
- finally {
- pageMem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, false, true);
- }
- }
- finally {
- pageMem.releasePage(fullId.groupId(), fullId.pageId(), page);
- }
- }
- finally {
- ignite0.context().cache().context().database().checkpointReadUnlock();
- }
- }
-
- ignite0.close();
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * Test recovery from WAL on 3 nodes in case of transactional cache.
- *
- * @throws Exception If fail.
- */
- public void testRecoveryOnTransactionalAndPartitionedCache() throws Exception {
- IgniteEx ignite = (IgniteEx) startGrids(3);
- ignite.active(true);
-
- try {
- final String cacheName = "transactional";
-
- CacheConfiguration<Object, Object> cacheConfiguration = new CacheConfiguration<>(cacheName)
- .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
- .setAffinity(new RendezvousAffinityFunction(false, 32))
- .setCacheMode(CacheMode.PARTITIONED)
- .setRebalanceMode(CacheRebalanceMode.SYNC)
- .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
- .setBackups(2);
-
- ignite.createCache(cacheConfiguration);
-
- IgniteCache<Object, Object> cache = ignite.cache(cacheName);
- Map<Object, Object> map = new HashMap<>();
-
- final int transactions = 100;
- final int operationsPerTransaction = 40;
-
- Random random = new Random();
-
- for (int t = 1; t <= transactions; t++) {
- Transaction tx = ignite.transactions().txStart(
- TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
-
- Map<Object, Object> changesInTransaction = new HashMap<>();
-
- for (int op = 0; op < operationsPerTransaction; op++) {
- int key = random.nextInt(1000) + 1;
-
- Object value;
- if (random.nextBoolean())
- value = randomString(random) + key;
- else
- value = new BigObject(key);
-
- changesInTransaction.put(key, value);
-
- cache.put(key, value);
- }
-
- if (random.nextBoolean()) {
- tx.commit();
- map.putAll(changesInTransaction);
- }
- else {
- tx.rollback();
- }
-
- if (t % 50 == 0)
- log.info("Finished transaction " + t);
- }
-
- stopAllGrids();
-
- ignite = (IgniteEx) startGrids(3);
- ignite.active(true);
-
- cache = ignite.cache(cacheName);
-
- for (Object key : map.keySet()) {
- Object expectedValue = map.get(key);
- Object actualValue = cache.get(key);
- Assert.assertEquals("Unexpected value for key " + key, expectedValue, actualValue);
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * Test that all DataRecord WAL records are within transaction boundaries - PREPARED and COMMITTED markers.
- *
- * @throws Exception If any fail.
- */
- public void testTxRecordsConsistency() throws Exception {
- System.setProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS, "true");
-
- IgniteEx ignite = (IgniteEx) startGrids(3);
- ignite.active(true);
-
- try {
- final String cacheName = "transactional";
-
- CacheConfiguration<Object, Object> cacheConfiguration = new CacheConfiguration<>(cacheName)
- .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
- .setAffinity(new RendezvousAffinityFunction(false, 32))
- .setCacheMode(CacheMode.PARTITIONED)
- .setRebalanceMode(CacheRebalanceMode.SYNC)
- .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
- .setBackups(0);
-
- ignite.createCache(cacheConfiguration);
-
- IgniteCache<Object, Object> cache = ignite.cache(cacheName);
-
- GridCacheSharedContext<Object, Object> sharedCtx = ignite.context().cache().context();
-
- GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)sharedCtx.database();
-
- db.waitForCheckpoint("test");
- db.enableCheckpoints(false).get();
-
- // Log something to know where to start.
- WALPointer startPtr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
-
- final int transactions = 100;
- final int operationsPerTransaction = 40;
-
- Random random = new Random();
-
- for (int t = 1; t <= transactions; t++) {
- Transaction tx = ignite.transactions().txStart(
- TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
-
- for (int op = 0; op < operationsPerTransaction; op++) {
- int key = random.nextInt(1000) + 1;
-
- Object value;
- if (random.nextBoolean())
- value = randomString(random) + key;
- else
- value = new BigObject(key);
-
- cache.put(key, value);
- }
-
- if (random.nextBoolean()) {
- tx.commit();
- }
- else {
- tx.rollback();
- }
-
- if (t % 50 == 0)
- log.info("Finished transaction " + t);
- }
-
- Set<GridCacheVersion> activeTransactions = new HashSet<>();
-
- // Check that all DataRecords are within PREPARED and COMMITTED tx records.
- try (WALIterator it = sharedCtx.wal().replay(startPtr)) {
- while (it.hasNext()) {
- IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
-
- WALRecord rec = tup.get2();
-
- if (rec instanceof TxRecord) {
- TxRecord txRecord = (TxRecord) rec;
- GridCacheVersion txId = txRecord.nearXidVersion();
-
- switch (txRecord.state()) {
- case PREPARED:
- assert !activeTransactions.contains(txId) : "Transaction is already present " + txRecord;
-
- activeTransactions.add(txId);
-
- break;
- case COMMITTED:
- assert activeTransactions.contains(txId) : "No PREPARE marker for transaction " + txRecord;
-
- activeTransactions.remove(txId);
-
- break;
- case ROLLED_BACK:
- activeTransactions.remove(txId);
- break;
-
- default:
- throw new IllegalStateException("Unknown Tx state of record " + txRecord);
- }
- } else if (rec instanceof DataRecord) {
- DataRecord dataRecord = (DataRecord) rec;
-
- for (DataEntry entry : dataRecord.writeEntries()) {
- GridCacheVersion txId = entry.nearXidVersion();
-
- assert activeTransactions.contains(txId) : "No transaction for entry " + entry;
- }
- }
- }
- }
- }
- finally {
- System.clearProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS);
- stopAllGrids();
- }
- }
-
- /**
- * Generate random lowercase string for test purposes.
- */
- private String randomString(Random random) {
- int len = random.nextInt(50) + 1;
-
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < len; i++)
- sb.append(random.nextInt(26) + 'a');
-
- return sb.toString();
- }
-
- /**
- * BigObject for test purposes that don't fit in page size.
- */
- private static class BigObject {
- private final int index;
-
- private final byte[] payload = new byte[4096];
-
- BigObject(int index) {
- this.index = index;
- // Create pseudo-random array.
- for (int i = 0; i < payload.length; i++)
- if (i % index == 0)
- payload[i] = (byte) index;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- BigObject bigObject = (BigObject) o;
- return index == bigObject.index &&
- Arrays.equals(payload, bigObject.payload);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(index, payload);
- }
- }
-
- /**
- * @param size Size of data.
- * @return Test data.
- */
- private int[] createTestData(int size) {
- int[] data = new int[size];
-
- for (int d = 0; d < size; ++d)
- data[d] = d;
-
- return data;
- }
-
- /**
- *
- */
- private static class LoadRunnable implements IgniteRunnable {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** */
- private boolean disableCheckpoints;
-
- /**
- * @param disableCheckpoints Disable checkpoints flag.
- */
- private LoadRunnable(boolean disableCheckpoints) {
- this.disableCheckpoints = disableCheckpoints;
- }
-
- /** {@inheritDoc} */
- @Override public void run() {
- ignite.log().info("Started load.");
-
- if (disableCheckpoints) {
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- try {
- dbMgr.enableCheckpoints(false).get();
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- try {
- boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- return ignite.cache("partitioned") != null;
- }
- }, 10_000);
-
- assertTrue(successfulWaiting);
- }
- catch (IgniteInterruptedCheckedException e) {
- throw new RuntimeException(e);
- }
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < 10_000; i++)
- cache.put(i, new IndexedObject(i));
-
- ignite.log().info("Finished load.");
- }
- }
-
- /**
- *
- */
- private static class AsyncLoadRunnable implements IgniteRunnable {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public void run() {
- try {
- boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- return ignite.cache("partitioned") != null;
- }
- }, 10_000);
-
- assertTrue(successfulWaiting);
- }
- catch (IgniteInterruptedCheckedException e) {
- throw new RuntimeException(e);
- }
-
- ignite.log().info(">>>>>>> Started load.");
-
- for (int i = 0; i < 4; i++) {
- ignite.scheduler().callLocal(new Callable<Object>() {
- @Override public Object call() throws Exception {
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- int cnt = 0;
-
- while (!Thread.currentThread().isInterrupted()) {
- cache.put(rnd.nextInt(10_000), new IndexedObject(rnd.nextInt()));
-
- cnt++;
-
- if (cnt > 0 && cnt % 1_000 == 0)
- ignite.log().info(">>>> Updated: " + cnt);
- }
-
- return null;
- }
- });
- }
- }
- }
-
- /**
- *
- */
- private static class VerifyCallable implements IgniteCallable<Boolean> {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public Boolean call() throws Exception {
- try {
- boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- return ignite.cache("partitioned") != null;
- }
- }, 10_000);
-
- assertTrue(successfulWaiting);
- }
- catch (IgniteInterruptedCheckedException e) {
- throw new RuntimeException(e);
- }
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < 10_000; i++) {
- Object val = cache.get(i);
-
- if (val == null) {
- ignite.log().warning("Failed to find a value for key: " + i);
-
- return false;
- }
- }
-
- return true;
- }
- }
-
- /**
- *
- */
- private static class LargeLoadRunnable implements IgniteRunnable {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** */
- private boolean disableCheckpoints;
-
- /**
- * @param disableCheckpoints Disable checkpoints flag.
- */
- private LargeLoadRunnable(boolean disableCheckpoints) {
- this.disableCheckpoints = disableCheckpoints;
- }
-
- /** {@inheritDoc} */
- @Override public void run() {
- try {
- boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- return ignite.cache("partitioned") != null;
- }
- }, 10_000);
-
- assertTrue(successfulWaiting);
- }
- catch (IgniteInterruptedCheckedException e) {
- throw new RuntimeException(e);
- }
-
- ignite.log().info("Started load.");
-
- if (disableCheckpoints) {
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- dbMgr.enableCheckpoints(false);
- }
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < 1000; i++) {
- final long[] data = new long[LARGE_ARR_SIZE];
-
- Arrays.fill(data, i);
-
- cache.put(i, data);
- }
-
- ignite.log().info("Finished load.");
- }
- }
-
- /**
- *
- */
- private static class AsyncLargeLoadRunnable implements IgniteRunnable {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public void run() {
- try {
- boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- return ignite.cache("partitioned") != null;
- }
- }, 10_000);
-
- assertTrue(successfulWaiting);
- }
- catch (IgniteInterruptedCheckedException e) {
- throw new RuntimeException(e);
- }
-
- ignite.log().info(">>>>>>> Started load.");
-
- for (int i = 0; i < 1; i++) {
- ignite.scheduler().callLocal(new Callable<Object>() {
- @Override public Object call() throws Exception {
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- int cnt = 0;
-
- while (!Thread.currentThread().isInterrupted()) {
- final long[] data = new long[LARGE_ARR_SIZE];
-
- final int key = rnd.nextInt(1000);
-
- Arrays.fill(data, key);
-
-// System.out.println("> " + key);
-
- cache.put(key, data);
-
- cnt++;
-
- if (cnt > 0 && cnt % 1_000 == 0)
- ignite.log().info(">>>> Updated: " + cnt);
- }
-
- return null;
- }
- });
- }
- }
- }
-
- /**
- *
- */
- private static class VerifyLargeCallable implements IgniteCallable<Boolean> {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public Boolean call() throws Exception {
- try {
- boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- return ignite.cache("partitioned") != null;
- }
- }, 10_000);
-
- assertTrue(successfulWaiting);
- }
- catch (IgniteInterruptedCheckedException e) {
- throw new RuntimeException(e);
- }
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < 1000; i++) {
- final long[] data = new long[LARGE_ARR_SIZE];
-
- Arrays.fill(data, i);
-
- final Object val = cache.get(i);
-
- if (val == null) {
- ignite.log().warning("Failed to find a value for key: " + i);
-
- return false;
- }
-
- assertTrue(Arrays.equals(data, (long[])val));
- }
-
- return true;
- }
- }
-
-
- /**
- *
- */
- private static class IndexedObject {
- /** */
- @QuerySqlField(index = true)
- private int iVal;
-
- /**
- * @param iVal Integer value.
- */
- private IndexedObject(int iVal) {
- this.iVal = iVal;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (!(o instanceof IndexedObject))
- return false;
-
- IndexedObject that = (IndexedObject)o;
-
- return iVal == that.iVal;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return iVal;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(IndexedObject.class, this);
- }
- }
-
- /**
- *
- */
- private enum EnumVal {
- /** */
- VAL1,
-
- /** */
- VAL2,
-
- /** */
- VAL3
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e9fd4e1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryWithCompactionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryWithCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryWithCompactionTest.java
deleted file mode 100644
index d927676..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryWithCompactionTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.ignite.internal.processors.cache.persistence.db.wal;
-
-import org.apache.ignite.configuration.IgniteConfiguration;
-
-/**
- *
- */
-public class IgniteWalRecoveryWithCompactionTest extends IgniteWalRecoveryTest {
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.getDataStorageConfiguration().setWalCompactionEnabled(true);
-
- return cfg;
- }
-}
[07/50] [abbrv] ignite git commit: IGNITE-9448 Update ZooKeeper
version to 3.4.13 - Fixes #4661.
Posted by ag...@apache.org.
IGNITE-9448 Update ZooKeeper version to 3.4.13 - Fixes #4661.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4d736fc2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4d736fc2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4d736fc2
Branch: refs/heads/ignite-5960
Commit: 4d736fc20a6e667e841201ad2205f1440546191b
Parents: 7e9fd4e
Author: vd-pyatkov <vp...@gridgain.com>
Authored: Fri Aug 31 18:41:58 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Aug 31 18:41:58 2018 +0300
----------------------------------------------------------------------
parent/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4d736fc2/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index 76e39b7..f5b49ba 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -122,7 +122,7 @@
<yammer.metrics.core.version>2.2.0</yammer.metrics.core.version>
<yardstick.version>0.8.3</yardstick.version>
<zkclient.version>0.5</zkclient.version>
- <zookeeper.version>3.4.6</zookeeper.version>
+ <zookeeper.version>3.4.13</zookeeper.version>
<!-- OSGI Manifest generation default property values -->
<osgi.import.package>*</osgi.import.package>
[23/50] [abbrv] ignite git commit: IGNITE-9387: [ML] Model updating
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
index 2f5d5d6..fb34c93 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
@@ -26,8 +26,10 @@ import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
import org.apache.ignite.ml.environment.LearningEnvironment;
+import org.apache.ignite.ml.environment.logging.MLLogger;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.primitives.vector.Vector;
+import org.jetbrains.annotations.NotNull;
/**
* Interface for trainers. Trainer is just a function which produces model from the data.
@@ -53,6 +55,71 @@ public abstract class DatasetTrainer<M extends Model, L> {
IgniteBiFunction<K, V, L> lbExtractor);
/**
+ * Gets state of model in arguments, compare it with training parameters of trainer and if they are fit then
+ * trainer updates model in according to new data and return new model. In other case trains new model.
+ *
+ * @param mdl Learned model.
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return Updated model.
+ */
+ public <K,V> M update(M mdl, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, L> lbExtractor) {
+
+ if(mdl != null) {
+ if(checkState(mdl)) {
+ return updateModel(mdl, datasetBuilder, featureExtractor, lbExtractor);
+ } else {
+ environment.logger(getClass()).log(
+ MLLogger.VerboseLevel.HIGH,
+ "Model cannot be updated because of initial state of " +
+ "it doesn't corresponds to trainer parameters"
+ );
+ }
+ }
+
+ return fit(datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /**
+ * @param mdl Model.
+ * @return true if current critical for training parameters correspond to parameters from last training.
+ */
+ protected abstract boolean checkState(M mdl);
+
+ /**
+ * Used on update phase when given dataset is empty.
+ * If last trained model exist then method returns it. In other case throws IllegalArgumentException.
+ *
+ * @param lastTrainedMdl Model.
+ */
+ @NotNull protected M getLastTrainedModelOrThrowEmptyDatasetException(M lastTrainedMdl) {
+ String msg = "Cannot train model on empty dataset";
+ if (lastTrainedMdl != null) {
+ environment.logger(getClass()).log(MLLogger.VerboseLevel.HIGH, msg);
+ return lastTrainedMdl;
+ } else
+ throw new EmptyDatasetException();
+ }
+
+ /**
+ * Gets state of model in arguments, update in according to new data and return new model.
+ *
+ * @param mdl Learned model.
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return Updated model.
+ */
+ protected abstract <K, V> M updateModel(M mdl, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, L> lbExtractor);
+
+ /**
* Trains model based on the specified data.
*
* @param ignite Ignite instance.
@@ -73,6 +140,27 @@ public abstract class DatasetTrainer<M extends Model, L> {
}
/**
+ * Gets state of model in arguments, update in according to new data and return new model.
+ *
+ * @param mdl Learned model.
+ * @param ignite Ignite instance.
+ * @param cache Ignite cache.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return Updated model.
+ */
+ public <K, V> M update(M mdl, Ignite ignite, IgniteCache<K, V> cache,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, L> lbExtractor) {
+ return update(
+ mdl, new CacheBasedDatasetBuilder<>(ignite, cache),
+ featureExtractor,
+ lbExtractor
+ );
+ }
+
+ /**
* Trains model based on the specified data.
*
* @param ignite Ignite instance.
@@ -94,6 +182,28 @@ public abstract class DatasetTrainer<M extends Model, L> {
}
/**
+ * Gets state of model in arguments, update in according to new data and return new model.
+ *
+ * @param mdl Learned model.
+ * @param ignite Ignite instance.
+ * @param cache Ignite cache.
+ * @param filter Filter for {@code upstream} data.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return Updated model.
+ */
+ public <K, V> M update(M mdl, Ignite ignite, IgniteCache<K, V> cache, IgniteBiPredicate<K, V> filter,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, L> lbExtractor) {
+ return update(
+ mdl, new CacheBasedDatasetBuilder<>(ignite, cache, filter),
+ featureExtractor,
+ lbExtractor
+ );
+ }
+
+ /**
* Trains model based on the specified data.
*
* @param data Data.
@@ -114,6 +224,27 @@ public abstract class DatasetTrainer<M extends Model, L> {
}
/**
+ * Gets state of model in arguments, update in according to new data and return new model.
+ *
+ * @param mdl Learned model.
+ * @param data Data.
+ * @param parts Number of partitions.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return Updated model.
+ */
+ public <K, V> M update(M mdl, Map<K, V> data, int parts, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, L> lbExtractor) {
+ return update(
+ mdl, new LocalDatasetBuilder<>(data, parts),
+ featureExtractor,
+ lbExtractor
+ );
+ }
+
+ /**
* Trains model based on the specified data.
*
* @param data Data.
@@ -136,10 +267,45 @@ public abstract class DatasetTrainer<M extends Model, L> {
}
/**
+ * Gets state of model in arguments, update in according to new data and return new model.
+ *
+ * @param data Data.
+ * @param filter Filter for {@code upstream} data.
+ * @param parts Number of partitions.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return Updated model.
+ */
+ public <K, V> M update(M mdl, Map<K, V> data, IgniteBiPredicate<K, V> filter, int parts,
+ IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, L> lbExtractor) {
+ return update(
+ mdl, new LocalDatasetBuilder<>(data, filter, parts),
+ featureExtractor,
+ lbExtractor
+ );
+ }
+
+ /**
* Sets learning Environment
* @param environment Environment.
*/
public void setEnvironment(LearningEnvironment environment) {
this.environment = environment;
}
+
+ /** */
+ public static class EmptyDatasetException extends IllegalArgumentException {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 6914650522523293521L;
+
+ /**
+ * Constructs an instance of EmptyDatasetException.
+ */
+ public EmptyDatasetException() {
+ super("Cannot train model on empty dataset");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
index de8994a..355048a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
@@ -86,6 +86,29 @@ public abstract class DecisionTree<T extends ImpurityMeasure<T>> extends Dataset
}
}
+ /**
+ * Trains new model based on dataset because there is no valid approach to update decision trees.
+ *
+ * @param mdl Learned model.
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return New model based on new dataset.
+ */
+ @Override public <K, V> DecisionTreeNode updateModel(DecisionTreeNode mdl, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ return fit(datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(DecisionTreeNode mdl) {
+ return true;
+ }
+
+ /** */
public <K,V> DecisionTreeNode fit(Dataset<EmptyContext, DecisionTreeData> dataset) {
return split(dataset, e -> true, 0, getImpurityMeasureCalculator(dataset));
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestClassifierTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestClassifierTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestClassifierTrainer.java
index 559dfff..7832584 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestClassifierTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestClassifierTrainer.java
@@ -64,8 +64,9 @@ public class RandomForestClassifierTrainer
* This id can be used as index in arrays or lists.
*
* @param dataset Dataset.
+ * @return true if initialization was done.
*/
- @Override protected void init(Dataset<EmptyContext, BootstrappedDatasetPartition> dataset) {
+ @Override protected boolean init(Dataset<EmptyContext, BootstrappedDatasetPartition> dataset) {
Set<Double> uniqLabels = dataset.compute(
x -> {
Set<Double> labels = new HashSet<>();
@@ -85,11 +86,14 @@ public class RandomForestClassifierTrainer
}
);
+ if(uniqLabels == null)
+ return false;
+
int i = 0;
for (Double label : uniqLabels)
lblMapping.put(label, i++);
- super.init(dataset);
+ return super.init(dataset);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java
index cb25aa3..91fcf0a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/randomforest/RandomForestTrainer.java
@@ -30,6 +30,7 @@ import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import org.apache.ignite.ml.Model;
import org.apache.ignite.ml.composition.ModelsComposition;
import org.apache.ignite.ml.dataset.Dataset;
import org.apache.ignite.ml.dataset.DatasetBuilder;
@@ -116,7 +117,8 @@ public abstract class RandomForestTrainer<L, S extends ImpurityComputer<Bootstra
new EmptyContextBuilder<>(),
new BootstrappedDatasetBuilder<>(featureExtractor, lbExtractor, cntOfTrees, subsampleSize))) {
- init(dataset);
+ if(!init(dataset))
+ return buildComposition(Collections.emptyList());
models = fit(dataset);
}
catch (Exception e) {
@@ -202,7 +204,8 @@ public abstract class RandomForestTrainer<L, S extends ImpurityComputer<Bootstra
*
* @param dataset Dataset.
*/
- protected void init(Dataset<EmptyContext, BootstrappedDatasetPartition> dataset) {
+ protected boolean init(Dataset<EmptyContext, BootstrappedDatasetPartition> dataset) {
+ return true;
}
/**
@@ -215,6 +218,8 @@ public abstract class RandomForestTrainer<L, S extends ImpurityComputer<Bootstra
Queue<TreeNode> treesQueue = createRootsQueue();
ArrayList<TreeRoot> roots = initTrees(treesQueue);
Map<Integer, BucketMeta> histMeta = computeHistogramMeta(meta, dataset);
+ if(histMeta.isEmpty())
+ return Collections.emptyList();
ImpurityHistogramsComputer<S> histogramsComputer = createImpurityHistogramsComputer();
while (!treesQueue.isEmpty()) {
@@ -232,6 +237,23 @@ public abstract class RandomForestTrainer<L, S extends ImpurityComputer<Bootstra
return roots;
}
+ /** {@inheritDoc} */
+ @Override protected boolean checkState(ModelsComposition mdl) {
+ ModelsComposition fakeComposition = buildComposition(Collections.emptyList());
+ return mdl.getPredictionsAggregator().getClass() == fakeComposition.getPredictionsAggregator().getClass();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected <K, V> ModelsComposition updateModel(ModelsComposition mdl, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ ArrayList<Model<Vector, Double>> oldModels = new ArrayList<>(mdl.getModels());
+ ModelsComposition newModels = fit(datasetBuilder, featureExtractor, lbExtractor);
+ oldModels.addAll(newModels.getModels());
+
+ return new ModelsComposition(oldModels, mdl.getPredictionsAggregator());
+ }
+
/**
* Split node with NodeId if need.
*
@@ -302,6 +324,8 @@ public abstract class RandomForestTrainer<L, S extends ImpurityComputer<Bootstra
List<NormalDistributionStatistics> stats = new NormalDistributionStatisticsComputer()
.computeStatistics(meta, dataset);
+ if(stats == null)
+ return Collections.emptyMap();
Map<Integer, BucketMeta> bucketsMeta = new HashMap<>();
for (int i = 0; i < stats.size(); i++) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java
index aae5af1..03f044a 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java
@@ -27,6 +27,7 @@ import org.apache.ignite.ml.math.distances.EuclideanDistance;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.math.primitives.vector.impl.DenseVector;
+import org.jetbrains.annotations.NotNull;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -39,19 +40,70 @@ public class KMeansTrainerTest {
/** Precision in test checks. */
private static final double PRECISION = 1e-2;
+ /** Data. */
+ private static final Map<Integer, double[]> data = new HashMap<>();
+
+ static {
+ data.put(0, new double[] {1.0, 1.0, 1.0});
+ data.put(1, new double[] {1.0, 2.0, 1.0});
+ data.put(2, new double[] {2.0, 1.0, 1.0});
+ data.put(3, new double[] {-1.0, -1.0, 2.0});
+ data.put(4, new double[] {-1.0, -2.0, 2.0});
+ data.put(5, new double[] {-2.0, -1.0, 2.0});
+ }
+
/**
* A few points, one cluster, one iteration
*/
@Test
public void findOneClusters() {
- Map<Integer, double[]> data = new HashMap<>();
- data.put(0, new double[]{1.0, 1.0, 1.0});
- data.put(1, new double[]{1.0, 2.0, 1.0});
- data.put(2, new double[]{2.0, 1.0, 1.0});
- data.put(3, new double[]{-1.0, -1.0, 2.0});
- data.put(4, new double[]{-1.0, -2.0, 2.0});
- data.put(5, new double[]{-2.0, -1.0, 2.0});
+ KMeansTrainer trainer = createAndCheckTrainer();
+ KMeansModel knnMdl = trainer.withK(1).fit(
+ new LocalDatasetBuilder<>(data, 2),
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[2]
+ );
+
+ Vector firstVector = new DenseVector(new double[] {2.0, 2.0});
+ assertEquals(knnMdl.apply(firstVector), 0.0, PRECISION);
+ Vector secondVector = new DenseVector(new double[] {-2.0, -2.0});
+ assertEquals(knnMdl.apply(secondVector), 0.0, PRECISION);
+ assertEquals(trainer.getMaxIterations(), 1);
+ assertEquals(trainer.getEpsilon(), PRECISION, PRECISION);
+ }
+ /** */
+ @Test
+ public void testUpdateMdl() {
+ KMeansTrainer trainer = createAndCheckTrainer();
+ KMeansModel originalMdl = trainer.withK(1).fit(
+ new LocalDatasetBuilder<>(data, 2),
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[2]
+ );
+ KMeansModel updatedMdlOnSameDataset = trainer.update(
+ originalMdl,
+ new LocalDatasetBuilder<>(data, 2),
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[2]
+ );
+ KMeansModel updatedMdlOnEmptyDataset = trainer.update(
+ originalMdl,
+ new LocalDatasetBuilder<>(new HashMap<Integer, double[]>(), 2),
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[2]
+ );
+
+ Vector firstVector = new DenseVector(new double[] {2.0, 2.0});
+ Vector secondVector = new DenseVector(new double[] {-2.0, -2.0});
+ assertEquals(originalMdl.apply(firstVector), updatedMdlOnSameDataset.apply(firstVector), PRECISION);
+ assertEquals(originalMdl.apply(secondVector), updatedMdlOnSameDataset.apply(secondVector), PRECISION);
+ assertEquals(originalMdl.apply(firstVector), updatedMdlOnEmptyDataset.apply(firstVector), PRECISION);
+ assertEquals(originalMdl.apply(secondVector), updatedMdlOnEmptyDataset.apply(secondVector), PRECISION);
+ }
+
+ /** */
+ @NotNull private KMeansTrainer createAndCheckTrainer() {
KMeansTrainer trainer = new KMeansTrainer()
.withDistance(new EuclideanDistance())
.withK(10)
@@ -61,20 +113,6 @@ public class KMeansTrainerTest {
assertEquals(10, trainer.getK());
assertEquals(2, trainer.getSeed());
assertTrue(trainer.getDistance() instanceof EuclideanDistance);
-
- KMeansModel knnMdl = trainer
- .withK(1)
- .fit(
- new LocalDatasetBuilder<>(data, 2),
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
- (k, v) -> v[2]
- );
-
- Vector firstVector = new DenseVector(new double[]{2.0, 2.0});
- assertEquals(knnMdl.apply(firstVector), 0.0, PRECISION);
- Vector secondVector = new DenseVector(new double[]{-2.0, -2.0});
- assertEquals(knnMdl.apply(secondVector), 0.0, PRECISION);
- assertEquals(trainer.getMaxIterations(), 1);
- assertEquals(trainer.getEpsilon(), PRECISION, PRECISION);
+ return trainer;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/common/CollectionsTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/common/CollectionsTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/common/CollectionsTest.java
index acf28e9..745eac9 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/common/CollectionsTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/common/CollectionsTest.java
@@ -22,6 +22,7 @@ import java.util.Set;
import org.apache.ignite.ml.clustering.kmeans.KMeansModel;
import org.apache.ignite.ml.clustering.kmeans.KMeansModelFormat;
import org.apache.ignite.ml.knn.ann.ANNClassificationModel;
+import org.apache.ignite.ml.knn.ann.ANNClassificationTrainer;
import org.apache.ignite.ml.knn.ann.ANNModelFormat;
import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
import org.apache.ignite.ml.knn.classification.KNNModelFormat;
@@ -103,11 +104,11 @@ public class CollectionsTest {
test(new SVMLinearBinaryClassificationModel(null, 1.0), new SVMLinearBinaryClassificationModel(null, 0.5));
- test(new ANNClassificationModel(new LabeledVectorSet<>()),
- new ANNClassificationModel(new LabeledVectorSet<>(1, 1, true)));
+ test(new ANNClassificationModel(new LabeledVectorSet<>(), new ANNClassificationTrainer.CentroidStat()),
+ new ANNClassificationModel(new LabeledVectorSet<>(1, 1, true), new ANNClassificationTrainer.CentroidStat()));
- test(new ANNModelFormat(1, new ManhattanDistance(), NNStrategy.SIMPLE, new LabeledVectorSet<>()),
- new ANNModelFormat(2, new ManhattanDistance(), NNStrategy.SIMPLE, new LabeledVectorSet<>()));
+ test(new ANNModelFormat(1, new ManhattanDistance(), NNStrategy.SIMPLE, new LabeledVectorSet<>(), new ANNClassificationTrainer.CentroidStat()),
+ new ANNModelFormat(2, new ManhattanDistance(), NNStrategy.SIMPLE, new LabeledVectorSet<>(), new ANNClassificationTrainer.CentroidStat()));
}
/** Test classes that have all instances equal (eg, metrics). */
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/common/LocalModelsTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/common/LocalModelsTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/common/LocalModelsTest.java
index 17d9c1a..9315850 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/common/LocalModelsTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/common/LocalModelsTest.java
@@ -32,6 +32,7 @@ import org.apache.ignite.ml.clustering.kmeans.KMeansTrainer;
import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
import org.apache.ignite.ml.knn.NNClassificationModel;
import org.apache.ignite.ml.knn.ann.ANNClassificationModel;
+import org.apache.ignite.ml.knn.ann.ANNClassificationTrainer;
import org.apache.ignite.ml.knn.ann.ANNModelFormat;
import org.apache.ignite.ml.knn.ann.ProbableLabel;
import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
@@ -237,7 +238,7 @@ public class LocalModelsTest {
executeModelTest(mdlFilePath -> {
final LabeledVectorSet<ProbableLabel, LabeledVector> centers = new LabeledVectorSet<>();
- NNClassificationModel mdl = new ANNClassificationModel(centers)
+ NNClassificationModel mdl = new ANNClassificationModel(centers, new ANNClassificationTrainer.CentroidStat())
.withK(4)
.withDistanceMeasure(new ManhattanDistance())
.withStrategy(NNStrategy.WEIGHTED);
@@ -250,7 +251,7 @@ public class LocalModelsTest {
Assert.assertNotNull(load);
- NNClassificationModel importedMdl = new ANNClassificationModel(load.getCandidates())
+ NNClassificationModel importedMdl = new ANNClassificationModel(load.getCandidates(), new ANNClassificationTrainer.CentroidStat())
.withK(load.getK())
.withDistanceMeasure(load.getDistanceMeasure())
.withStrategy(load.getStgy());
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java
index 4452668..3e340f6 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/composition/boosting/GDBTrainerTest.java
@@ -54,7 +54,7 @@ public class GDBTrainerTest {
learningSample.put(i, new double[] {xs[i], ys[i]});
}
- DatasetTrainer<Model<Vector, Double>, Double> trainer
+ DatasetTrainer<ModelsComposition, Double> trainer
= new GDBRegressionOnTreesTrainer(1.0, 2000, 3, 0.0).withUseIndex(true);
Model<Vector, Double> mdl = trainer.fit(
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/knn/ANNClassificationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/ANNClassificationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/ANNClassificationTest.java
index 7289b1d..d8fb620 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/ANNClassificationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/ANNClassificationTest.java
@@ -26,6 +26,7 @@ import org.apache.ignite.ml.knn.ann.ANNClassificationModel;
import org.apache.ignite.ml.knn.ann.ANNClassificationTrainer;
import org.apache.ignite.ml.knn.classification.NNStrategy;
import org.apache.ignite.ml.math.distances.EuclideanDistance;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -68,4 +69,47 @@ public class ANNClassificationTest extends TrainerTest {
Assert.assertTrue(mdl.toString(true).contains(NNStrategy.SIMPLE.name()));
Assert.assertTrue(mdl.toString(false).contains(NNStrategy.SIMPLE.name()));
}
+
+ /** */
+ @Test
+ public void testUpdate() {
+ Map<Integer, double[]> cacheMock = new HashMap<>();
+
+ for (int i = 0; i < twoClusters.length; i++)
+ cacheMock.put(i, twoClusters[i]);
+
+ ANNClassificationTrainer trainer = new ANNClassificationTrainer()
+ .withK(10)
+ .withMaxIterations(10)
+ .withEpsilon(1e-4)
+ .withDistance(new EuclideanDistance());
+
+ ANNClassificationModel originalMdl = (ANNClassificationModel) trainer.withSeed(1234L).fit(
+ cacheMock,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ ).withK(3)
+ .withDistanceMeasure(new EuclideanDistance())
+ .withStrategy(NNStrategy.SIMPLE);
+
+ ANNClassificationModel updatedOnSameDataset = trainer.withSeed(1234L).update(originalMdl,
+ cacheMock, parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[2]
+ );
+
+ ANNClassificationModel updatedOnEmptyDataset = trainer.withSeed(1234L).update(originalMdl,
+ new HashMap<Integer, double[]>(), parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[2]
+ );
+
+ Vector v1 = VectorUtils.of(550, 550);
+ Vector v2 = VectorUtils.of(-550, -550);
+ TestUtils.assertEquals(originalMdl.apply(v1), updatedOnSameDataset.apply(v1), PRECISION);
+ TestUtils.assertEquals(originalMdl.apply(v2), updatedOnSameDataset.apply(v2), PRECISION);
+ TestUtils.assertEquals(originalMdl.apply(v1), updatedOnEmptyDataset.apply(v1), PRECISION);
+ TestUtils.assertEquals(originalMdl.apply(v2), updatedOnEmptyDataset.apply(v2), PRECISION);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
index c5a5c1c..748123a 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
@@ -174,4 +174,43 @@ public class KNNClassificationTest {
Vector vector = new DenseVector(new double[] {-1.01, -1.01});
assertEquals(knnMdl.apply(vector), 1.0);
}
+
+ /** */
+ @Test
+ public void testUpdate() {
+ Map<Integer, double[]> data = new HashMap<>();
+ data.put(0, new double[] {10.0, 10.0, 1.0});
+ data.put(1, new double[] {10.0, 20.0, 1.0});
+ data.put(2, new double[] {-1, -1, 1.0});
+ data.put(3, new double[] {-2, -2, 2.0});
+ data.put(4, new double[] {-1.0, -2.0, 2.0});
+ data.put(5, new double[] {-2.0, -1.0, 2.0});
+
+ KNNClassificationTrainer trainer = new KNNClassificationTrainer();
+
+ KNNClassificationModel originalMdl = (KNNClassificationModel)trainer.fit(
+ data,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[2]
+ ).withK(3)
+ .withDistanceMeasure(new EuclideanDistance())
+ .withStrategy(NNStrategy.WEIGHTED);
+
+ KNNClassificationModel updatedOnSameDataset = trainer.update(originalMdl,
+ data, parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[2]
+ );
+
+ KNNClassificationModel updatedOnEmptyDataset = trainer.update(originalMdl,
+ new HashMap<Integer, double[]>(), parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[2]
+ );
+
+ Vector vector = new DenseVector(new double[] {-1.01, -1.01});
+ assertEquals(originalMdl.apply(vector), updatedOnSameDataset.apply(vector));
+ assertEquals(originalMdl.apply(vector), updatedOnEmptyDataset.apply(vector));
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
index 5504e1a..52ff1ec 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
@@ -35,6 +35,8 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import static junit.framework.TestCase.assertEquals;
+
/**
* Tests for {@link KNNRegressionTrainer}.
*/
@@ -135,4 +137,42 @@ public class KNNRegressionTest {
Assert.assertTrue(knnMdl.toString(true).contains(stgy.name()));
Assert.assertTrue(knnMdl.toString(false).contains(stgy.name()));
}
+
+ /** */
+ @Test
+ public void testUpdate() {
+ Map<Integer, double[]> data = new HashMap<>();
+ data.put(0, new double[] {11.0, 0, 0, 0, 0, 0});
+ data.put(1, new double[] {12.0, 2.0, 0, 0, 0, 0});
+ data.put(2, new double[] {13.0, 0, 3.0, 0, 0, 0});
+ data.put(3, new double[] {14.0, 0, 0, 4.0, 0, 0});
+ data.put(4, new double[] {15.0, 0, 0, 0, 5.0, 0});
+ data.put(5, new double[] {16.0, 0, 0, 0, 0, 6.0});
+
+ KNNRegressionTrainer trainer = new KNNRegressionTrainer();
+
+ KNNRegressionModel originalMdl = (KNNRegressionModel) trainer.fit(
+ new LocalDatasetBuilder<>(data, parts),
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ ).withK(1)
+ .withDistanceMeasure(new EuclideanDistance())
+ .withStrategy(NNStrategy.SIMPLE);
+
+ KNNRegressionModel updatedOnSameDataset = trainer.update(originalMdl,
+ data, parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[2]
+ );
+
+ KNNRegressionModel updatedOnEmptyDataset = trainer.update(originalMdl,
+ new HashMap<Integer, double[]>(), parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[2]
+ );
+
+ Vector vector = new DenseVector(new double[] {0, 0, 0, 5.0, 0.0});
+ assertEquals(originalMdl.apply(vector), updatedOnSameDataset.apply(vector));
+ assertEquals(originalMdl.apply(vector), updatedOnEmptyDataset.apply(vector));
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
index a1d601c..6a6555e 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
@@ -29,6 +29,7 @@ import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.math.primitives.vector.impl.DenseVector;
import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
import org.apache.ignite.ml.optimization.LossFunctions;
+import org.apache.ignite.ml.optimization.SmoothParametrized;
import org.apache.ignite.ml.optimization.updatecalculators.NesterovParameterUpdate;
import org.apache.ignite.ml.optimization.updatecalculators.NesterovUpdateCalculator;
import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
@@ -154,6 +155,69 @@ public class MLPTrainerTest {
TestUtils.checkIsInEpsilonNeighbourhood(new DenseVector(new double[]{0.0}), predict.getRow(0), 1E-1);
}
+
+ /** */
+ @Test
+ public void testUpdate() {
+ UpdatesStrategy<SmoothParametrized, SimpleGDParameterUpdate> updatesStgy = new UpdatesStrategy<>(
+ new SimpleGDUpdateCalculator(0.2),
+ SimpleGDParameterUpdate::sumLocal,
+ SimpleGDParameterUpdate::avg
+ );
+
+ Map<Integer, double[][]> xorData = new HashMap<>();
+ xorData.put(0, new double[][]{{0.0, 0.0}, {0.0}});
+ xorData.put(1, new double[][]{{0.0, 1.0}, {1.0}});
+ xorData.put(2, new double[][]{{1.0, 0.0}, {1.0}});
+ xorData.put(3, new double[][]{{1.0, 1.0}, {0.0}});
+
+ MLPArchitecture arch = new MLPArchitecture(2).
+ withAddedLayer(10, true, Activators.RELU).
+ withAddedLayer(1, false, Activators.SIGMOID);
+
+ MLPTrainer<SimpleGDParameterUpdate> trainer = new MLPTrainer<>(
+ arch,
+ LossFunctions.MSE,
+ updatesStgy,
+ 3000,
+ batchSize,
+ 50,
+ 123L
+ );
+
+ MultilayerPerceptron originalMdl = trainer.fit(
+ xorData,
+ parts,
+ (k, v) -> VectorUtils.of(v[0]),
+ (k, v) -> v[1]
+ );
+
+ MultilayerPerceptron updatedOnSameDS = trainer.update(
+ originalMdl,
+ xorData,
+ parts,
+ (k, v) -> VectorUtils.of(v[0]),
+ (k, v) -> v[1]
+ );
+
+ MultilayerPerceptron updatedOnEmptyDS = trainer.update(
+ originalMdl,
+ new HashMap<Integer, double[][]>(),
+ parts,
+ (k, v) -> VectorUtils.of(v[0]),
+ (k, v) -> v[1]
+ );
+
+ DenseMatrix matrix = new DenseMatrix(new double[][] {
+ {0.0, 0.0},
+ {0.0, 1.0},
+ {1.0, 0.0},
+ {1.0, 1.0}
+ });
+
+ TestUtils.checkIsInEpsilonNeighbourhood(originalMdl.apply(matrix).getRow(0), updatedOnSameDS.apply(matrix).getRow(0), 1E-1);
+ TestUtils.checkIsInEpsilonNeighbourhood(originalMdl.apply(matrix).getRow(0), updatedOnEmptyDS.apply(matrix).getRow(0), 1E-1);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java
index d16ae72..9c35ac7 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java
@@ -101,4 +101,55 @@ public class LinearRegressionLSQRTrainerTest extends TrainerTest {
assertEquals(intercept, mdl.getIntercept(), 1e-6);
}
+
+ /** */
+ @Test
+ public void testUpdate() {
+ Random rnd = new Random(0);
+ Map<Integer, double[]> data = new HashMap<>();
+ double[] coef = new double[100];
+ double intercept = rnd.nextDouble() * 10;
+
+ for (int i = 0; i < 100000; i++) {
+ double[] x = new double[coef.length + 1];
+
+ for (int j = 0; j < coef.length; j++)
+ x[j] = rnd.nextDouble() * 10;
+
+ x[coef.length] = intercept;
+
+ data.put(i, x);
+ }
+
+ LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
+
+ LinearRegressionModel originalModel = trainer.fit(
+ data,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[coef.length]
+ );
+
+ LinearRegressionModel updatedOnSameDS = trainer.update(
+ originalModel,
+ data,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[coef.length]
+ );
+
+ LinearRegressionModel updatedOnEmpyDS = trainer.update(
+ originalModel,
+ new HashMap<Integer, double[]>(),
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[coef.length]
+ );
+
+ assertArrayEquals(originalModel.getWeights().getStorage().data(), updatedOnSameDS.getWeights().getStorage().data(), 1e-6);
+ assertEquals(originalModel.getIntercept(), updatedOnSameDS.getIntercept(), 1e-6);
+
+ assertArrayEquals(originalModel.getWeights().getStorage().data(), updatedOnEmpyDS.getWeights().getStorage().data(), 1e-6);
+ assertEquals(originalModel.getIntercept(), updatedOnEmpyDS.getIntercept(), 1e-6);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
index 349e712..86b0f27 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
@@ -72,4 +72,66 @@ public class LinearRegressionSGDTrainerTest extends TrainerTest {
assertEquals(2.8421709430404007e-14, mdl.getIntercept(), 1e-1);
}
+
+ /** */
+ @Test
+ public void testUpdate() {
+ Map<Integer, double[]> data = new HashMap<>();
+ data.put(0, new double[]{-1.0915526, 1.81983527, -0.91409478, 0.70890712, -24.55724107});
+ data.put(1, new double[]{-0.61072904, 0.37545517, 0.21705352, 0.09516495, -26.57226867});
+ data.put(2, new double[]{0.05485406, 0.88219898, -0.80584547, 0.94668307, 61.80919728});
+ data.put(3, new double[]{-0.24835094, -0.34000053, -1.69984651, -1.45902635, -161.65525991});
+ data.put(4, new double[]{0.63675392, 0.31675535, 0.38837437, -1.1221971, -14.46432611});
+ data.put(5, new double[]{0.14194017, 2.18158997, -0.28397346, -0.62090588, -3.2122197});
+ data.put(6, new double[]{-0.53487507, 1.4454797, 0.21570443, -0.54161422, -46.5469012});
+ data.put(7, new double[]{-1.58812173, -0.73216803, -2.15670676, -1.03195988, -247.23559889});
+ data.put(8, new double[]{0.20702671, 0.92864654, 0.32721202, -0.09047503, 31.61484949});
+ data.put(9, new double[]{-0.37890345, -0.04846179, -0.84122753, -1.14667474, -124.92598583});
+
+ LinearRegressionSGDTrainer<?> trainer = new LinearRegressionSGDTrainer<>(new UpdatesStrategy<>(
+ new RPropUpdateCalculator(),
+ RPropParameterUpdate::sumLocal,
+ RPropParameterUpdate::avg
+ ), 100000, 10, 100, 0L);
+
+ LinearRegressionModel originalModel = trainer.withSeed(0).fit(
+ data,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[4]
+ );
+
+
+ LinearRegressionModel updatedOnSameDS = trainer.withSeed(0).update(
+ originalModel,
+ data,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[4]
+ );
+
+ LinearRegressionModel updatedOnEmptyDS = trainer.withSeed(0).update(
+ originalModel,
+ new HashMap<Integer, double[]>(),
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[4]
+ );
+
+ assertArrayEquals(
+ originalModel.getWeights().getStorage().data(),
+ updatedOnSameDS.getWeights().getStorage().data(),
+ 1.0
+ );
+
+ assertEquals(originalModel.getIntercept(), updatedOnSameDS.getIntercept(), 1.0);
+
+ assertArrayEquals(
+ originalModel.getWeights().getStorage().data(),
+ updatedOnEmptyDS.getWeights().getStorage().data(),
+ 1e-1
+ );
+
+ assertEquals(originalModel.getIntercept(), updatedOnEmptyDS.getIntercept(), 1e-1);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java
index 1f8c5d1..f08501c 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java
@@ -19,9 +19,11 @@ package org.apache.ignite.ml.regressions.logistic;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.ignite.ml.TestUtils;
import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.nn.UpdatesStrategy;
import org.apache.ignite.ml.optimization.SmoothParametrized;
@@ -81,4 +83,60 @@ public class LogRegMultiClassTrainerTest extends TrainerTest {
TestUtils.assertEquals(2, mdl.apply(VectorUtils.of(-10, -10)), PRECISION);
TestUtils.assertEquals(3, mdl.apply(VectorUtils.of(10, -10)), PRECISION);
}
+
+ /** */
+ @Test
+ public void testUpdate() {
+ Map<Integer, double[]> cacheMock = new HashMap<>();
+
+ for (int i = 0; i < fourSetsInSquareVertices.length; i++)
+ cacheMock.put(i, fourSetsInSquareVertices[i]);
+
+ LogRegressionMultiClassTrainer<?> trainer = new LogRegressionMultiClassTrainer<>()
+ .withUpdatesStgy(new UpdatesStrategy<>(
+ new SimpleGDUpdateCalculator(0.2),
+ SimpleGDParameterUpdate::sumLocal,
+ SimpleGDParameterUpdate::avg
+ ))
+ .withAmountOfIterations(1000)
+ .withAmountOfLocIterations(10)
+ .withBatchSize(100)
+ .withSeed(123L);
+
+ LogRegressionMultiClassModel originalModel = trainer.fit(
+ cacheMock,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
+
+ LogRegressionMultiClassModel updatedOnSameDS = trainer.update(
+ originalModel,
+ cacheMock,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
+
+ LogRegressionMultiClassModel updatedOnEmptyDS = trainer.update(
+ originalModel,
+ new HashMap<Integer, double[]>(),
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
+
+ List<Vector> vectors = Arrays.asList(
+ VectorUtils.of(10, 10),
+ VectorUtils.of(-10, 10),
+ VectorUtils.of(-10, -10),
+ VectorUtils.of(10, -10)
+ );
+
+
+ for (Vector vec : vectors) {
+ TestUtils.assertEquals(originalModel.apply(vec), updatedOnSameDS.apply(vec), PRECISION);
+ TestUtils.assertEquals(originalModel.apply(vec), updatedOnEmptyDS.apply(vec), PRECISION);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainerTest.java
index 5bd2dbd..1da0d1a 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainerTest.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.ignite.ml.TestUtils;
import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.nn.UpdatesStrategy;
import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpdate;
@@ -60,4 +61,49 @@ public class LogisticRegressionSGDTrainerTest extends TrainerTest {
TestUtils.assertEquals(0, mdl.apply(VectorUtils.of(100, 10)), PRECISION);
TestUtils.assertEquals(1, mdl.apply(VectorUtils.of(10, 100)), PRECISION);
}
+
+ /** */
+ @Test
+ public void testUpdate() {
+ Map<Integer, double[]> cacheMock = new HashMap<>();
+
+ for (int i = 0; i < twoLinearlySeparableClasses.length; i++)
+ cacheMock.put(i, twoLinearlySeparableClasses[i]);
+
+ LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
+ new SimpleGDUpdateCalculator().withLearningRate(0.2),
+ SimpleGDParameterUpdate::sumLocal,
+ SimpleGDParameterUpdate::avg
+ ), 100000, 10, 100, 123L);
+
+ LogisticRegressionModel originalModel = trainer.fit(
+ cacheMock,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
+
+ LogisticRegressionModel updatedOnSameDS = trainer.update(
+ originalModel,
+ cacheMock,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
+
+ LogisticRegressionModel updatedOnEmptyDS = trainer.update(
+ originalModel,
+ new HashMap<Integer, double[]>(),
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
+
+ Vector v1 = VectorUtils.of(100, 10);
+ Vector v2 = VectorUtils.of(10, 100);
+ TestUtils.assertEquals(originalModel.apply(v1), updatedOnSameDS.apply(v1), PRECISION);
+ TestUtils.assertEquals(originalModel.apply(v2), updatedOnSameDS.apply(v2), PRECISION);
+ TestUtils.assertEquals(originalModel.apply(v2), updatedOnEmptyDS.apply(v2), PRECISION);
+ TestUtils.assertEquals(originalModel.apply(v1), updatedOnEmptyDS.apply(v1), PRECISION);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
index 5630bee..263bb6d 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.ignite.ml.TestUtils;
import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.junit.Test;
@@ -52,4 +53,44 @@ public class SVMBinaryTrainerTest extends TrainerTest {
TestUtils.assertEquals(-1, mdl.apply(VectorUtils.of(100, 10)), PRECISION);
TestUtils.assertEquals(1, mdl.apply(VectorUtils.of(10, 100)), PRECISION);
}
+
+ /** */
+ @Test
+ public void testUpdate() {
+ Map<Integer, double[]> cacheMock = new HashMap<>();
+
+ for (int i = 0; i < twoLinearlySeparableClasses.length; i++)
+ cacheMock.put(i, twoLinearlySeparableClasses[i]);
+
+ SVMLinearBinaryClassificationTrainer trainer = new SVMLinearBinaryClassificationTrainer()
+ .withAmountOfIterations(1000)
+ .withSeed(1234L);
+
+ SVMLinearBinaryClassificationModel originalModel = trainer.fit(
+ cacheMock,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
+
+ SVMLinearBinaryClassificationModel updatedOnSameDS = trainer.update(
+ originalModel,
+ cacheMock,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
+
+ SVMLinearBinaryClassificationModel updatedOnEmptyDS = trainer.update(
+ originalModel,
+ new HashMap<Integer, double[]>(),
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
+
+ Vector v = VectorUtils.of(100, 10);
+ TestUtils.assertEquals(originalModel.apply(v), updatedOnSameDS.apply(v), PRECISION);
+ TestUtils.assertEquals(originalModel.apply(v), updatedOnEmptyDS.apply(v), PRECISION);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java
index 7ea28c2..e0c62af 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.ignite.ml.TestUtils;
import org.apache.ignite.ml.common.TrainerTest;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.junit.Test;
@@ -54,4 +55,46 @@ public class SVMMultiClassTrainerTest extends TrainerTest {
TestUtils.assertEquals(0, mdl.apply(VectorUtils.of(100, 10)), PRECISION);
TestUtils.assertEquals(1, mdl.apply(VectorUtils.of(10, 100)), PRECISION);
}
+
+ /** */
+ @Test
+ public void testUpdate() {
+ Map<Integer, double[]> cacheMock = new HashMap<>();
+
+ for (int i = 0; i < twoLinearlySeparableClasses.length; i++)
+ cacheMock.put(i, twoLinearlySeparableClasses[i]);
+
+ SVMLinearMultiClassClassificationTrainer trainer = new SVMLinearMultiClassClassificationTrainer()
+ .withLambda(0.3)
+ .withAmountOfLocIterations(10)
+ .withAmountOfIterations(100)
+ .withSeed(1234L);
+
+ SVMLinearMultiClassClassificationModel originalModel = trainer.fit(
+ cacheMock,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
+
+ SVMLinearMultiClassClassificationModel updatedOnSameDS = trainer.update(
+ originalModel,
+ cacheMock,
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
+
+ SVMLinearMultiClassClassificationModel updatedOnEmptyDS = trainer.update(
+ originalModel,
+ new HashMap<Integer, double[]>(),
+ parts,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
+
+ Vector v = VectorUtils.of(100, 10);
+ TestUtils.assertEquals(originalModel.apply(v), updatedOnSameDS.apply(v), PRECISION);
+ TestUtils.assertEquals(originalModel.apply(v), updatedOnEmptyDS.apply(v), PRECISION);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestClassifierTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestClassifierTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestClassifierTrainerTest.java
index 4abf508..087f4e8 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestClassifierTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestClassifierTrainerTest.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.ignite.ml.composition.ModelsComposition;
import org.apache.ignite.ml.composition.predictionsaggregator.OnMajorityPredictionsAggregator;
import org.apache.ignite.ml.dataset.feature.FeatureMeta;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -75,7 +76,7 @@ public class RandomForestClassifierTrainerTest {
}
ArrayList<FeatureMeta> meta = new ArrayList<>();
- for(int i = 0; i < 4; i++)
+ for (int i = 0; i < 4; i++)
meta.add(new FeatureMeta("", i, false));
RandomForestClassifierTrainer trainer = new RandomForestClassifierTrainer(meta)
.withCountOfTrees(5)
@@ -86,4 +87,34 @@ public class RandomForestClassifierTrainerTest {
assertTrue(mdl.getPredictionsAggregator() instanceof OnMajorityPredictionsAggregator);
assertEquals(5, mdl.getModels().size());
}
+
+ /** */
+ @Test
+ public void testUpdate() {
+ int sampleSize = 1000;
+ Map<double[], Double> sample = new HashMap<>();
+ for (int i = 0; i < sampleSize; i++) {
+ double x1 = i;
+ double x2 = x1 / 10.0;
+ double x3 = x2 / 10.0;
+ double x4 = x3 / 10.0;
+
+ sample.put(new double[] {x1, x2, x3, x4}, (double)(i % 2));
+ }
+
+ ArrayList<FeatureMeta> meta = new ArrayList<>();
+ for (int i = 0; i < 4; i++)
+ meta.add(new FeatureMeta("", i, false));
+ RandomForestClassifierTrainer trainer = new RandomForestClassifierTrainer(meta)
+ .withCountOfTrees(100)
+ .withFeaturesCountSelectionStrgy(x -> 2);
+
+ ModelsComposition originalModel = trainer.fit(sample, parts, (k, v) -> VectorUtils.of(k), (k, v) -> v);
+ ModelsComposition updatedOnSameDS = trainer.update(originalModel, sample, parts, (k, v) -> VectorUtils.of(k), (k, v) -> v);
+ ModelsComposition updatedOnEmptyDS = trainer.update(originalModel, new HashMap<double[], Double>(), parts, (k, v) -> VectorUtils.of(k), (k, v) -> v);
+
+ Vector v = VectorUtils.of(5, 0.5, 0.05, 0.005);
+ assertEquals(originalModel.apply(v), updatedOnSameDS.apply(v), 0.01);
+ assertEquals(originalModel.apply(v), updatedOnEmptyDS.apply(v), 0.01);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4c18f11/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestRegressionTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestRegressionTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestRegressionTrainerTest.java
index c4a4a75..fcc20bd 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestRegressionTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/randomforest/RandomForestRegressionTrainerTest.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.ignite.ml.composition.ModelsComposition;
import org.apache.ignite.ml.composition.predictionsaggregator.MeanValuePredictionsAggregator;
import org.apache.ignite.ml.dataset.feature.FeatureMeta;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -82,4 +83,34 @@ public class RandomForestRegressionTrainerTest {
assertTrue(mdl.getPredictionsAggregator() instanceof MeanValuePredictionsAggregator);
assertEquals(5, mdl.getModels().size());
}
+
+ /** */
+ @Test
+ public void testUpdate() {
+ int sampleSize = 1000;
+ Map<double[], Double> sample = new HashMap<>();
+ for (int i = 0; i < sampleSize; i++) {
+ double x1 = i;
+ double x2 = x1 / 10.0;
+ double x3 = x2 / 10.0;
+ double x4 = x3 / 10.0;
+
+ sample.put(new double[] {x1, x2, x3, x4}, (double)(i % 2));
+ }
+
+ ArrayList<FeatureMeta> meta = new ArrayList<>();
+ for (int i = 0; i < 4; i++)
+ meta.add(new FeatureMeta("", i, false));
+ RandomForestRegressionTrainer trainer = new RandomForestRegressionTrainer(meta)
+ .withCountOfTrees(100)
+ .withFeaturesCountSelectionStrgy(x -> 2);
+
+ ModelsComposition originalModel = trainer.fit(sample, parts, (k, v) -> VectorUtils.of(k), (k, v) -> v);
+ ModelsComposition updatedOnSameDS = trainer.update(originalModel, sample, parts, (k, v) -> VectorUtils.of(k), (k, v) -> v);
+ ModelsComposition updatedOnEmptyDS = trainer.update(originalModel, new HashMap<double[], Double>(), parts, (k, v) -> VectorUtils.of(k), (k, v) -> v);
+
+ Vector v = VectorUtils.of(5, 0.5, 0.05, 0.005);
+ assertEquals(originalModel.apply(v), updatedOnSameDS.apply(v), 0.1);
+ assertEquals(originalModel.apply(v), updatedOnEmptyDS.apply(v), 0.1);
+ }
}
[47/50] [abbrv] ignite git commit: IGNITE-9366: SQL: Added
IGNITE.NODE_METRICS system view. This closes #4615.
Posted by ag...@apache.org.
IGNITE-9366: SQL: Added IGNITE.NODE_METRICS system view. This closes #4615.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/fd3c50e0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/fd3c50e0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/fd3c50e0
Branch: refs/heads/ignite-5960
Commit: fd3c50e0df916cd636b98849d02d3a730fcac73d
Parents: b10ba04
Author: Aleksey Plekhanov <pl...@gmail.com>
Authored: Fri Sep 7 14:22:48 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Sep 7 14:22:48 2018 +0300
----------------------------------------------------------------------
.../processors/query/h2/IgniteH2Indexing.java | 2 +
.../h2/sys/view/SqlAbstractLocalSystemView.java | 27 +++
.../h2/sys/view/SqlSystemViewNodeMetrics.java | 210 +++++++++++++++++
.../query/SqlSystemViewsSelfTest.java | 226 +++++++++++++++++--
4 files changed, 448 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/fd3c50e0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 9b71f37..a795552 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -130,6 +130,7 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable;
import org.apache.ignite.internal.processors.query.h2.sys.SqlSystemTableEngine;
import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewBaselineNodes;
import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodeAttributes;
+import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodeMetrics;
import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodes;
import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor;
import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor;
@@ -3202,6 +3203,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
views.add(new SqlSystemViewNodes(ctx));
views.add(new SqlSystemViewNodeAttributes(ctx));
views.add(new SqlSystemViewBaselineNodes(ctx));
+ views.add(new SqlSystemViewNodeMetrics(ctx));
return views;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/fd3c50e0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
index ac90b63..d692dba 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
@@ -26,6 +26,8 @@ import org.h2.table.Column;
import org.h2.value.Value;
import org.h2.value.ValueNull;
import org.h2.value.ValueString;
+import org.h2.value.ValueTime;
+import org.h2.value.ValueTimestamp;
/**
* Local system view base class (which uses only local node data).
@@ -125,4 +127,29 @@ public abstract class SqlAbstractLocalSystemView extends SqlAbstractSystemView {
return null;
}
}
+
+ /**
+ * Converts millis to ValueTime
+ *
+ * @param millis Millis.
+ */
+ protected static Value valueTimeFromMillis(long millis) {
+ if (millis == -1L || millis == Long.MAX_VALUE)
+ return ValueNull.INSTANCE;
+ else
+ // Note: ValueTime.fromMillis(long) method trying to convert time using timezone and return wrong result.
+ return ValueTime.fromNanos(millis * 1_000_000L);
+ }
+
+ /**
+ * Converts millis to ValueTimestamp
+ *
+ * @param millis Millis.
+ */
+ protected static Value valueTimestampFromMillis(long millis) {
+ if (millis <= 0L || millis == Long.MAX_VALUE)
+ return ValueNull.INSTANCE;
+ else
+ return ValueTimestamp.fromMillis(millis);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/fd3c50e0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java
new file mode 100644
index 0000000..01b4e97
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys.view;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.util.typedef.F;
+import org.h2.engine.Session;
+import org.h2.result.Row;
+import org.h2.result.SearchRow;
+import org.h2.value.Value;
+
+/**
+ * System view: node metrics.
+ */
+public class SqlSystemViewNodeMetrics extends SqlAbstractLocalSystemView {
+ /**
+ * @param ctx Grid context.
+ */
+ public SqlSystemViewNodeMetrics(GridKernalContext ctx) {
+ super("NODE_METRICS", "Node metrics", ctx, new String[] {"NODE_ID"},
+ newColumn("NODE_ID", Value.UUID),
+ newColumn("LAST_UPDATE_TIME", Value.TIMESTAMP),
+ newColumn("MAX_ACTIVE_JOBS", Value.INT),
+ newColumn("CUR_ACTIVE_JOBS", Value.INT),
+ newColumn("AVG_ACTIVE_JOBS", Value.FLOAT),
+ newColumn("MAX_WAITING_JOBS", Value.INT),
+ newColumn("CUR_WAITING_JOBS", Value.INT),
+ newColumn("AVG_WAITING_JOBS", Value.FLOAT),
+ newColumn("MAX_REJECTED_JOBS", Value.INT),
+ newColumn("CUR_REJECTED_JOBS", Value.INT),
+ newColumn("AVG_REJECTED_JOBS", Value.FLOAT),
+ newColumn("TOTAL_REJECTED_JOBS", Value.INT),
+ newColumn("MAX_CANCELED_JOBS", Value.INT),
+ newColumn("CUR_CANCELED_JOBS", Value.INT),
+ newColumn("AVG_CANCELED_JOBS", Value.FLOAT),
+ newColumn("TOTAL_CANCELED_JOBS", Value.INT),
+ newColumn("MAX_JOBS_WAIT_TIME", Value.TIME),
+ newColumn("CUR_JOBS_WAIT_TIME", Value.TIME),
+ newColumn("AVG_JOBS_WAIT_TIME", Value.TIME),
+ newColumn("MAX_JOBS_EXECUTE_TIME", Value.TIME),
+ newColumn("CUR_JOBS_EXECUTE_TIME", Value.TIME),
+ newColumn("AVG_JOBS_EXECUTE_TIME", Value.TIME),
+ newColumn("TOTAL_JOBS_EXECUTE_TIME", Value.TIME),
+ newColumn("TOTAL_EXECUTED_JOBS", Value.INT),
+ newColumn("TOTAL_EXECUTED_TASKS", Value.INT),
+ newColumn("TOTAL_BUSY_TIME", Value.TIME),
+ newColumn("TOTAL_IDLE_TIME", Value.TIME),
+ newColumn("CUR_IDLE_TIME", Value.TIME),
+ newColumn("BUSY_TIME_PERCENTAGE", Value.FLOAT),
+ newColumn("IDLE_TIME_PERCENTAGE", Value.FLOAT),
+ newColumn("TOTAL_CPU", Value.INT),
+ newColumn("CUR_CPU_LOAD", Value.DOUBLE),
+ newColumn("AVG_CPU_LOAD", Value.DOUBLE),
+ newColumn("CUR_GC_CPU_LOAD", Value.DOUBLE),
+ newColumn("HEAP_MEMORY_INIT", Value.LONG),
+ newColumn("HEAP_MEMORY_USED", Value.LONG),
+ newColumn("HEAP_MEMORY_COMMITED", Value.LONG),
+ newColumn("HEAP_MEMORY_MAX", Value.LONG),
+ newColumn("HEAP_MEMORY_TOTAL", Value.LONG),
+ newColumn("NONHEAP_MEMORY_INIT", Value.LONG),
+ newColumn("NONHEAP_MEMORY_USED", Value.LONG),
+ newColumn("NONHEAP_MEMORY_COMMITED", Value.LONG),
+ newColumn("NONHEAP_MEMORY_MAX", Value.LONG),
+ newColumn("NONHEAP_MEMORY_TOTAL", Value.LONG),
+ newColumn("UPTIME", Value.TIME),
+ newColumn("JVM_START_TIME", Value.TIMESTAMP),
+ newColumn("NODE_START_TIME", Value.TIMESTAMP),
+ newColumn("LAST_DATA_VERSION", Value.LONG),
+ newColumn("CUR_THREAD_COUNT", Value.INT),
+ newColumn("MAX_THREAD_COUNT", Value.INT),
+ newColumn("TOTAL_THREAD_COUNT", Value.LONG),
+ newColumn("CUR_DAEMON_THREAD_COUNT", Value.INT),
+ newColumn("SENT_MESSAGES_COUNT", Value.INT),
+ newColumn("SENT_BYTES_COUNT", Value.LONG),
+ newColumn("RECEIVED_MESSAGES_COUNT", Value.INT),
+ newColumn("RECEIVED_BYTES_COUNT", Value.LONG),
+ newColumn("OUTBOUND_MESSAGES_QUEUE", Value.INT)
+ );
+ }
+
+ /** {@inheritDoc} */
+ @Override public Iterator<Row> getRows(Session ses, SearchRow first, SearchRow last) {
+ List<Row> rows = new ArrayList<>();
+
+ Collection<ClusterNode> nodes;
+
+ SqlSystemViewColumnCondition idCond = conditionForColumn("NODE_ID", first, last);
+
+ if (idCond.isEquality()) {
+ try {
+ UUID nodeId = uuidFromValue(idCond.valueForEquality());
+
+ ClusterNode node = nodeId == null ? null : ctx.discovery().node(nodeId);
+
+ if (node != null)
+ nodes = Collections.singleton(node);
+ else
+ nodes = Collections.emptySet();
+ }
+ catch (Exception e) {
+ nodes = Collections.emptySet();
+ }
+ }
+ else
+ nodes = F.concat(false, ctx.discovery().allNodes(), ctx.discovery().daemonNodes());
+
+ for (ClusterNode node : nodes) {
+ if (node != null) {
+ ClusterMetrics metrics = node.metrics();
+
+ rows.add(
+ createRow(ses, rows.size(),
+ node.id(),
+ valueTimestampFromMillis(metrics.getLastUpdateTime()),
+ metrics.getMaximumActiveJobs(),
+ metrics.getCurrentActiveJobs(),
+ metrics.getAverageActiveJobs(),
+ metrics.getMaximumWaitingJobs(),
+ metrics.getCurrentWaitingJobs(),
+ metrics.getAverageWaitingJobs(),
+ metrics.getMaximumRejectedJobs(),
+ metrics.getCurrentRejectedJobs(),
+ metrics.getAverageRejectedJobs(),
+ metrics.getTotalRejectedJobs(),
+ metrics.getMaximumCancelledJobs(),
+ metrics.getCurrentCancelledJobs(),
+ metrics.getAverageCancelledJobs(),
+ metrics.getTotalCancelledJobs(),
+ valueTimeFromMillis(metrics.getMaximumJobWaitTime()),
+ valueTimeFromMillis(metrics.getCurrentJobWaitTime()),
+ valueTimeFromMillis((long)metrics.getAverageJobWaitTime()),
+ valueTimeFromMillis(metrics.getMaximumJobExecuteTime()),
+ valueTimeFromMillis(metrics.getCurrentJobExecuteTime()),
+ valueTimeFromMillis((long)metrics.getAverageJobExecuteTime()),
+ valueTimeFromMillis(metrics.getTotalJobsExecutionTime()),
+ metrics.getTotalExecutedJobs(),
+ metrics.getTotalExecutedTasks(),
+ valueTimeFromMillis(metrics.getTotalBusyTime()),
+ valueTimeFromMillis(metrics.getTotalIdleTime()),
+ valueTimeFromMillis(metrics.getCurrentIdleTime()),
+ metrics.getBusyTimePercentage(),
+ metrics.getIdleTimePercentage(),
+ metrics.getTotalCpus(),
+ metrics.getCurrentCpuLoad(),
+ metrics.getAverageCpuLoad(),
+ metrics.getCurrentGcCpuLoad(),
+ metrics.getHeapMemoryInitialized(),
+ metrics.getHeapMemoryUsed(),
+ metrics.getHeapMemoryCommitted(),
+ metrics.getHeapMemoryMaximum(),
+ metrics.getHeapMemoryTotal(),
+ metrics.getNonHeapMemoryInitialized(),
+ metrics.getNonHeapMemoryUsed(),
+ metrics.getNonHeapMemoryCommitted(),
+ metrics.getNonHeapMemoryMaximum(),
+ metrics.getNonHeapMemoryTotal(),
+ valueTimeFromMillis(metrics.getUpTime()),
+ valueTimestampFromMillis(metrics.getStartTime()),
+ valueTimestampFromMillis(metrics.getNodeStartTime()),
+ metrics.getLastDataVersion(),
+ metrics.getCurrentThreadCount(),
+ metrics.getMaximumThreadCount(),
+ metrics.getTotalStartedThreadCount(),
+ metrics.getCurrentDaemonThreadCount(),
+ metrics.getSentMessagesCount(),
+ metrics.getSentBytesCount(),
+ metrics.getReceivedMessagesCount(),
+ metrics.getReceivedBytesCount(),
+ metrics.getOutboundMessagesQueueSize()
+ )
+ );
+ }
+ }
+
+ return rows.iterator();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean canGetRowCount() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getRowCount() {
+ return F.concat(false, ctx.discovery().allNodes(), ctx.discovery().daemonNodes()).size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/fd3c50e0/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
index c8374a7..1a4dae7 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
@@ -17,21 +17,31 @@
package org.apache.ignite.internal.processors.query;
+import java.sql.Time;
+import java.sql.Timestamp;
import java.util.Collections;
import java.util.List;
+import java.util.Random;
+import java.util.TimeZone;
import java.util.UUID;
import java.util.concurrent.Callable;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.QueryEntity;
import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.ClusterMetricsSnapshot;
import org.apache.ignite.internal.IgniteNodeAttributes;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteRunnable;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -39,6 +49,9 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
* Tests for ignite SQL system views.
*/
public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
+ /** Metrics check attempts. */
+ private static final int METRICS_CHECK_ATTEMPTS = 10;
+
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
super.beforeTest();
@@ -171,7 +184,7 @@ public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
private void assertColumnTypes(List<?> rowData, Class<?> ... colTypes) {
for (int i = 0; i < colTypes.length; i++) {
if (rowData.get(i) != null)
- assertEquals("Column " + i + " type", rowData.get(i).getClass(), colTypes[i]);
+ assertEquals("Column " + i + " type", colTypes[i], rowData.get(i).getClass());
}
}
@@ -181,9 +194,14 @@ public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
public void testNodesViews() throws Exception {
- Ignite ignite1 = startGrid(getTestIgniteInstanceName(), getConfiguration());
- Ignite ignite2 = startGrid(getTestIgniteInstanceName(1), getConfiguration().setClientMode(true));
- Ignite ignite3 = startGrid(getTestIgniteInstanceName(2), getConfiguration().setDaemon(true));
+ Ignite igniteSrv = startGrid(getTestIgniteInstanceName(), getConfiguration().setMetricsUpdateFrequency(500L));
+
+ Ignite igniteCli = startGrid(getTestIgniteInstanceName(1), getConfiguration().setMetricsUpdateFrequency(500L)
+ .setClientMode(true));
+
+ startGrid(getTestIgniteInstanceName(2), getConfiguration().setMetricsUpdateFrequency(500L).setDaemon(true));
+
+ UUID nodeId0 = igniteSrv.cluster().localNode().id();
awaitPartitionMapExchange();
@@ -201,7 +219,7 @@ public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
assertEquals(1, resSrv.size());
- assertEquals(ignite1.cluster().localNode().id(), resSrv.get(0).get(0));
+ assertEquals(nodeId0, resSrv.get(0).get(0));
assertEquals(1, resSrv.get(0).get(1));
@@ -210,7 +228,7 @@ public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
assertEquals(1, resCli.size());
- assertEquals(ignite2.cluster().localNode().id(), resCli.get(0).get(0));
+ assertEquals(nodeId(1), resCli.get(0).get(0));
assertEquals(2, resCli.get(0).get(1));
@@ -219,7 +237,7 @@ public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
assertEquals(1, resDaemon.size());
- assertEquals(ignite3.cluster().localNode().id(), resDaemon.get(0).get(0));
+ assertEquals(nodeId(2), resDaemon.get(0).get(0));
assertEquals(3, resDaemon.get(0).get(1));
@@ -227,32 +245,30 @@ public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
assertEquals(0, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = '-'").size());
assertEquals(1, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = ?",
- ignite1.cluster().localNode().id()).size());
+ nodeId0).size());
assertEquals(1, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = ?",
- ignite3.cluster().localNode().id()).size());
+ nodeId(2)).size());
// Check index on ID column with disjunction.
assertEquals(3, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = ? " +
- "OR node_order=1 OR node_order=2 OR node_order=3", ignite1.cluster().localNode().id()).size());
+ "OR node_order=1 OR node_order=2 OR node_order=3", nodeId0).size());
// Check quick-count.
assertEquals(3L, execSql("SELECT COUNT(*) FROM IGNITE.NODES").get(0).get(0));
// Check joins
- assertEquals(ignite1.cluster().localNode().id(), execSql("SELECT N1.ID FROM IGNITE.NODES N1 JOIN " +
+ assertEquals(nodeId0, execSql("SELECT N1.ID FROM IGNITE.NODES N1 JOIN " +
"IGNITE.NODES N2 ON N1.NODE_ORDER = N2.NODE_ORDER JOIN IGNITE.NODES N3 ON N2.ID = N3.ID " +
"WHERE N3.NODE_ORDER = 1")
.get(0).get(0));
// Check sub-query
- assertEquals(ignite1.cluster().localNode().id(), execSql("SELECT N1.ID FROM IGNITE.NODES N1 " +
+ assertEquals(nodeId0, execSql("SELECT N1.ID FROM IGNITE.NODES N1 " +
"WHERE NOT EXISTS (SELECT 1 FROM IGNITE.NODES N2 WHERE N2.ID = N1.ID AND N2.NODE_ORDER <> 1)")
.get(0).get(0));
// Check node attributes view
- UUID cliNodeId = ignite2.cluster().localNode().id();
-
String cliAttrName = IgniteNodeAttributes.ATTR_CLIENT_MODE;
assertColumnTypes(execSql("SELECT NODE_ID, NAME, VALUE FROM IGNITE.NODE_ATTRIBUTES").get(0),
@@ -267,7 +283,7 @@ public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
assertEquals(1,
execSql("SELECT NODE_ID FROM IGNITE.NODE_ATTRIBUTES WHERE NODE_ID = ? AND NAME = ? AND VALUE = 'true'",
- cliNodeId, cliAttrName).size());
+ nodeId(1), cliAttrName).size());
assertEquals(0,
execSql("SELECT NODE_ID FROM IGNITE.NODE_ATTRIBUTES WHERE NODE_ID = '-' AND NAME = ?",
@@ -275,7 +291,169 @@ public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
assertEquals(0,
execSql("SELECT NODE_ID FROM IGNITE.NODE_ATTRIBUTES WHERE NODE_ID = ? AND NAME = '-'",
- cliNodeId).size());
+ nodeId(1)).size());
+
+ // Check node metrics view.
+ String sqlAllMetrics = "SELECT NODE_ID, LAST_UPDATE_TIME, " +
+ "MAX_ACTIVE_JOBS, CUR_ACTIVE_JOBS, AVG_ACTIVE_JOBS, " +
+ "MAX_WAITING_JOBS, CUR_WAITING_JOBS, AVG_WAITING_JOBS, " +
+ "MAX_REJECTED_JOBS, CUR_REJECTED_JOBS, AVG_REJECTED_JOBS, TOTAL_REJECTED_JOBS, " +
+ "MAX_CANCELED_JOBS, CUR_CANCELED_JOBS, AVG_CANCELED_JOBS, TOTAL_CANCELED_JOBS, " +
+ "MAX_JOBS_WAIT_TIME, CUR_JOBS_WAIT_TIME, AVG_JOBS_WAIT_TIME, " +
+ "MAX_JOBS_EXECUTE_TIME, CUR_JOBS_EXECUTE_TIME, AVG_JOBS_EXECUTE_TIME, TOTAL_JOBS_EXECUTE_TIME, " +
+ "TOTAL_EXECUTED_JOBS, TOTAL_EXECUTED_TASKS, " +
+ "TOTAL_BUSY_TIME, TOTAL_IDLE_TIME, CUR_IDLE_TIME, BUSY_TIME_PERCENTAGE, IDLE_TIME_PERCENTAGE, " +
+ "TOTAL_CPU, CUR_CPU_LOAD, AVG_CPU_LOAD, CUR_GC_CPU_LOAD, " +
+ "HEAP_MEMORY_INIT, HEAP_MEMORY_USED, HEAP_MEMORY_COMMITED, HEAP_MEMORY_MAX, HEAP_MEMORY_TOTAL, " +
+ "NONHEAP_MEMORY_INIT, NONHEAP_MEMORY_USED, NONHEAP_MEMORY_COMMITED, NONHEAP_MEMORY_MAX, NONHEAP_MEMORY_TOTAL, " +
+ "UPTIME, JVM_START_TIME, NODE_START_TIME, LAST_DATA_VERSION, " +
+ "CUR_THREAD_COUNT, MAX_THREAD_COUNT, TOTAL_THREAD_COUNT, CUR_DAEMON_THREAD_COUNT, " +
+ "SENT_MESSAGES_COUNT, SENT_BYTES_COUNT, RECEIVED_MESSAGES_COUNT, RECEIVED_BYTES_COUNT, " +
+ "OUTBOUND_MESSAGES_QUEUE FROM IGNITE.NODE_METRICS";
+
+ List<List<?>> resMetrics = execSql(sqlAllMetrics);
+
+ assertColumnTypes(resMetrics.get(0), UUID.class, Timestamp.class,
+ Integer.class, Integer.class, Float.class, // Active jobs.
+ Integer.class, Integer.class, Float.class, // Waiting jobs.
+ Integer.class, Integer.class, Float.class, Integer.class, // Rejected jobs.
+ Integer.class, Integer.class, Float.class, Integer.class, // Canceled jobs.
+ Time.class, Time.class, Time.class, // Jobs wait time.
+ Time.class, Time.class, Time.class, Time.class, // Jobs execute time.
+ Integer.class, Integer.class, // Executed jobs/task.
+ Time.class, Time.class, Time.class, Float.class, Float.class, // Busy/idle time.
+ Integer.class, Double.class, Double.class, Double.class, // CPU.
+ Long.class, Long.class, Long.class, Long.class, Long.class, // Heap memory.
+ Long.class, Long.class, Long.class, Long.class, Long.class, // Nonheap memory.
+ Time.class, Timestamp.class, Timestamp.class, Long.class, // Uptime.
+ Integer.class, Integer.class, Long.class, Integer.class, // Threads.
+ Integer.class, Long.class, Integer.class, Long.class, // Sent/received messages.
+ Integer.class); // Outbound message queue.
+
+ assertEquals(3, resAll.size());
+
+ // Check join with nodes.
+ assertEquals(3, execSql("SELECT NM.LAST_UPDATE_TIME FROM IGNITE.NODES N " +
+ "JOIN IGNITE.NODE_METRICS NM ON N.ID = NM.NODE_ID").size());
+
+ // Check index on NODE_ID column.
+ assertEquals(1, execSql("SELECT LAST_UPDATE_TIME FROM IGNITE.NODE_METRICS WHERE NODE_ID = ?",
+ nodeId(1)).size());
+
+ // Check malformed value for indexed column.
+ assertEquals(0, execSql("SELECT LAST_UPDATE_TIME FROM IGNITE.NODE_METRICS WHERE NODE_ID = ?",
+ "-").size());
+
+ // Check quick-count.
+ assertEquals(3L, execSql("SELECT COUNT(*) FROM IGNITE.NODE_METRICS").get(0).get(0));
+
+ // Check metric values.
+
+ // Broadcast jobs to server and client nodes to get non zero metric values.
+ for (int i = 0; i < 100; i++) {
+ IgniteFuture<Void > fut = igniteSrv.compute(igniteSrv.cluster().forNodeId(nodeId0, nodeId(1)))
+ .broadcastAsync(
+ new IgniteRunnable() {
+ @Override public void run() {
+ Random rnd = new Random();
+
+ try {
+ doSleep(rnd.nextInt(100));
+ }
+ catch (Throwable ignore) {
+ // No-op.
+ }
+ }
+ });
+
+ if (i % 10 == 0)
+ fut.cancel();
+ }
+
+ doSleep(igniteSrv.configuration().getMetricsUpdateFrequency() * 3L);
+
+ for (Ignite grid : G.allGrids()) {
+ UUID nodeId = grid.cluster().localNode().id();
+
+ // Metrics for node must be collected from another node to avoid race and get consistent metrics snapshot.
+ Ignite ignite = F.eq(nodeId, nodeId0) ? igniteCli : igniteSrv;
+
+ for (int i = 0; i < METRICS_CHECK_ATTEMPTS; i++) {
+ ClusterMetrics metrics = ignite.cluster().node(nodeId).metrics();
+
+ assertTrue(metrics instanceof ClusterMetricsSnapshot);
+
+ resMetrics = execSql(ignite, sqlAllMetrics + " WHERE NODE_ID = ?", nodeId);
+
+ log.info("Check metrics for node " + grid.name() + ", attempt " + (i + 1));
+
+ if (metrics.getLastUpdateTime() == ((Timestamp)resMetrics.get(0).get(1)).getTime()) {
+ assertEquals(metrics.getMaximumActiveJobs(), resMetrics.get(0).get(2));
+ assertEquals(metrics.getCurrentActiveJobs(), resMetrics.get(0).get(3));
+ assertEquals(metrics.getAverageActiveJobs(), resMetrics.get(0).get(4));
+ assertEquals(metrics.getMaximumWaitingJobs(), resMetrics.get(0).get(5));
+ assertEquals(metrics.getCurrentWaitingJobs(), resMetrics.get(0).get(6));
+ assertEquals(metrics.getAverageWaitingJobs(), resMetrics.get(0).get(7));
+ assertEquals(metrics.getMaximumRejectedJobs(), resMetrics.get(0).get(8));
+ assertEquals(metrics.getCurrentRejectedJobs(), resMetrics.get(0).get(9));
+ assertEquals(metrics.getAverageRejectedJobs(), resMetrics.get(0).get(10));
+ assertEquals(metrics.getTotalRejectedJobs(), resMetrics.get(0).get(11));
+ assertEquals(metrics.getMaximumCancelledJobs(), resMetrics.get(0).get(12));
+ assertEquals(metrics.getCurrentCancelledJobs(), resMetrics.get(0).get(13));
+ assertEquals(metrics.getAverageCancelledJobs(), resMetrics.get(0).get(14));
+ assertEquals(metrics.getTotalCancelledJobs(), resMetrics.get(0).get(15));
+ assertEquals(metrics.getMaximumJobWaitTime(), convertToMilliseconds(resMetrics.get(0).get(16)));
+ assertEquals(metrics.getCurrentJobWaitTime(), convertToMilliseconds(resMetrics.get(0).get(17)));
+ assertEquals((long)metrics.getAverageJobWaitTime(), convertToMilliseconds(resMetrics.get(0).get(18)));
+ assertEquals(metrics.getMaximumJobExecuteTime(), convertToMilliseconds(resMetrics.get(0).get(19)));
+ assertEquals(metrics.getCurrentJobExecuteTime(), convertToMilliseconds(resMetrics.get(0).get(20)));
+ assertEquals((long)metrics.getAverageJobExecuteTime(), convertToMilliseconds(resMetrics.get(0).get(21)));
+ assertEquals(metrics.getTotalJobsExecutionTime(), convertToMilliseconds(resMetrics.get(0).get(22)));
+ assertEquals(metrics.getTotalExecutedJobs(), resMetrics.get(0).get(23));
+ assertEquals(metrics.getTotalExecutedTasks(), resMetrics.get(0).get(24));
+ assertEquals(metrics.getTotalBusyTime(), convertToMilliseconds(resMetrics.get(0).get(25)));
+ assertEquals(metrics.getTotalIdleTime(), convertToMilliseconds(resMetrics.get(0).get(26)));
+ assertEquals(metrics.getCurrentIdleTime(), convertToMilliseconds(resMetrics.get(0).get(27)));
+ assertEquals(metrics.getBusyTimePercentage(), resMetrics.get(0).get(28));
+ assertEquals(metrics.getIdleTimePercentage(), resMetrics.get(0).get(29));
+ assertEquals(metrics.getTotalCpus(), resMetrics.get(0).get(30));
+ assertEquals(metrics.getCurrentCpuLoad(), resMetrics.get(0).get(31));
+ assertEquals(metrics.getAverageCpuLoad(), resMetrics.get(0).get(32));
+ assertEquals(metrics.getCurrentGcCpuLoad(), resMetrics.get(0).get(33));
+ assertEquals(metrics.getHeapMemoryInitialized(), resMetrics.get(0).get(34));
+ assertEquals(metrics.getHeapMemoryUsed(), resMetrics.get(0).get(35));
+ assertEquals(metrics.getHeapMemoryCommitted(), resMetrics.get(0).get(36));
+ assertEquals(metrics.getHeapMemoryMaximum(), resMetrics.get(0).get(37));
+ assertEquals(metrics.getHeapMemoryTotal(), resMetrics.get(0).get(38));
+ assertEquals(metrics.getNonHeapMemoryInitialized(), resMetrics.get(0).get(39));
+ assertEquals(metrics.getNonHeapMemoryUsed(), resMetrics.get(0).get(40));
+ assertEquals(metrics.getNonHeapMemoryCommitted(), resMetrics.get(0).get(41));
+ assertEquals(metrics.getNonHeapMemoryMaximum(), resMetrics.get(0).get(42));
+ assertEquals(metrics.getNonHeapMemoryTotal(), resMetrics.get(0).get(43));
+ assertEquals(metrics.getUpTime(), convertToMilliseconds(resMetrics.get(0).get(44)));
+ assertEquals(metrics.getStartTime(), ((Timestamp)resMetrics.get(0).get(45)).getTime());
+ assertEquals(metrics.getNodeStartTime(), ((Timestamp)resMetrics.get(0).get(46)).getTime());
+ assertEquals(metrics.getLastDataVersion(), resMetrics.get(0).get(47));
+ assertEquals(metrics.getCurrentThreadCount(), resMetrics.get(0).get(48));
+ assertEquals(metrics.getMaximumThreadCount(), resMetrics.get(0).get(49));
+ assertEquals(metrics.getTotalStartedThreadCount(), resMetrics.get(0).get(50));
+ assertEquals(metrics.getCurrentDaemonThreadCount(), resMetrics.get(0).get(51));
+ assertEquals(metrics.getSentMessagesCount(), resMetrics.get(0).get(52));
+ assertEquals(metrics.getSentBytesCount(), resMetrics.get(0).get(53));
+ assertEquals(metrics.getReceivedMessagesCount(), resMetrics.get(0).get(54));
+ assertEquals(metrics.getReceivedBytesCount(), resMetrics.get(0).get(55));
+ assertEquals(metrics.getOutboundMessagesQueueSize(), resMetrics.get(0).get(56));
+
+ break;
+ }
+ else {
+ log.info("Metrics was updated in background, will retry check");
+
+ if (i == METRICS_CHECK_ATTEMPTS - 1)
+ fail("Failed to check metrics, attempts limit reached (" + METRICS_CHECK_ATTEMPTS + ')');
+ }
+ }
+ }
}
/**
@@ -322,7 +500,7 @@ public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
}
/**
- * Gets ignite configuration with persistance enabled.
+ * Gets ignite configuration with persistence enabled.
*/
private IgniteConfiguration getPdsConfiguration(String consistentId) throws Exception {
IgniteConfiguration cfg = getConfiguration();
@@ -336,4 +514,18 @@ public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
return cfg;
}
+
+ /**
+ * Convert Time to milliseconds.
+ *
+ * Note: Returned Time values from SQL it's milliseconds since January 1, 1970, 00:00:00 GMT. To get right interval
+ * in milliseconds this value must be adjusted to current time zone.
+ *
+ * @param sqlTime Time value returned from SQL.
+ */
+ private long convertToMilliseconds(Object sqlTime) {
+ Time time0 = (Time)sqlTime;
+
+ return time0.getTime() + TimeZone.getDefault().getOffset(time0.getTime());
+ }
}
[48/50] [abbrv] ignite git commit: IGNITE-9161: Optimization for C++
(copy elision)
Posted by ag...@apache.org.
IGNITE-9161: Optimization for C++ (copy elision)
This closes #4476
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/40432b09
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/40432b09
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/40432b09
Branch: refs/heads/ignite-5960
Commit: 40432b093f400ae505869cca8a1f5ab745bf5206
Parents: fd3c50e
Author: Igor Sapego <is...@apache.org>
Authored: Fri Sep 7 15:16:02 2018 +0300
Committer: Igor Sapego <is...@apache.org>
Committed: Fri Sep 7 15:16:02 2018 +0300
----------------------------------------------------------------------
.../include/ignite/binary/binary_raw_reader.h | 22 ++-
.../ignite/impl/binary/binary_reader_impl.h | 11 ++
.../ignite/impl/binary/binary_type_impl.h | 16 +-
.../cpp/core/include/ignite/cache/cache.h | 46 ++++--
.../include/ignite/cache/query/query_cursor.h | 15 +-
.../cpp/core/include/ignite/impl/operations.h | 150 ++++++-------------
.../cpp/odbc-test/src/connection_test.cpp | 7 +-
7 files changed, 128 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/40432b09/modules/platforms/cpp/binary/include/ignite/binary/binary_raw_reader.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/binary/include/ignite/binary/binary_raw_reader.h b/modules/platforms/cpp/binary/include/ignite/binary/binary_raw_reader.h
index c06cb91..73d2525 100644
--- a/modules/platforms/cpp/binary/include/ignite/binary/binary_raw_reader.h
+++ b/modules/platforms/cpp/binary/include/ignite/binary/binary_raw_reader.h
@@ -313,18 +313,30 @@ namespace ignite
*/
std::string ReadString()
{
+ std::string res;
+
+ ReadString(res);
+
+ return res;
+ }
+
+ /**
+ * Read string from the stream.
+ *
+ * @param dst String.
+ */
+ void ReadString(std::string& dst)
+ {
int32_t len = ReadString(NULL, 0);
if (len != -1)
{
- ignite::common::FixedSizeArray<char> arr(len + 1);
-
- ReadString(arr.GetData(), static_cast<int32_t>(arr.GetSize()));
+ dst.resize(static_cast<size_t>(len));
- return std::string(arr.GetData());
+ ReadString(&dst[0], len);
}
else
- return std::string();
+ dst.clear();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/40432b09/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h b/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h
index 94f5ec5..a3b880c 100644
--- a/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h
+++ b/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_reader_impl.h
@@ -902,6 +902,17 @@ namespace ignite
/**
* Read object.
*
+ * @param res Read object.
+ */
+ template<typename T>
+ void ReadTopObject(T& res)
+ {
+ return ignite::binary::ReadHelper<T>::Read(*this, res);
+ }
+
+ /**
+ * Read object.
+ *
* @return Read object.
*/
template<typename R, typename T>
http://git-wip-us.apache.org/repos/asf/ignite/blob/40432b09/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_type_impl.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_type_impl.h b/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_type_impl.h
index 64638c2..e9bbace 100644
--- a/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_type_impl.h
+++ b/modules/platforms/cpp/binary/include/ignite/impl/binary/binary_type_impl.h
@@ -97,10 +97,16 @@ namespace ignite
{
T res;
- reader.template ReadTopObject0<ignite::binary::BinaryReader, T>(res);
+ Read<R>(reader, res);
return res;
}
+
+ template<typename R>
+ static void Read(R& reader, T& val)
+ {
+ reader.template ReadTopObject0<BinaryReader, T>(val);
+ }
};
/**
@@ -117,10 +123,16 @@ namespace ignite
std::auto_ptr<T> res(new T());
- reader.template ReadTopObject0<ignite::binary::BinaryReader, T>(*res);
+ reader.template ReadTopObject0<BinaryReader, T>(*res);
return res.release();
}
+
+ template<typename R>
+ static void Read(R& reader, T*& ptr)
+ {
+ ptr = Read<R>(reader);
+ }
};
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/40432b09/modules/platforms/cpp/core/include/ignite/cache/cache.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/core/include/ignite/cache/cache.h b/modules/platforms/cpp/core/include/ignite/cache/cache.h
index b16d7f5..c230361 100644
--- a/modules/platforms/cpp/core/include/ignite/cache/cache.h
+++ b/modules/platforms/cpp/core/include/ignite/cache/cache.h
@@ -259,12 +259,14 @@ namespace ignite
*/
V LocalPeek(const K& key, int32_t peekModes, IgniteError& err)
{
+ V val;
+
impl::InCacheLocalPeekOperation<K> inOp(key, peekModes);
- impl::Out1Operation<V> outOp;
+ impl::Out1Operation<V> outOp(val);
impl.Get()->LocalPeek(inOp, outOp, peekModes, err);
- return outOp.GetResult();
+ return val;
}
/**
@@ -305,12 +307,13 @@ namespace ignite
*/
V Get(const K& key, IgniteError& err)
{
+ V val;
impl::In1Operation<K> inOp(key);
- impl::Out1Operation<V> outOp;
+ impl::Out1Operation<V> outOp(val);
impl.Get()->Get(inOp, outOp, err);
- return outOp.GetResult();
+ return val;
}
/**
@@ -351,12 +354,14 @@ namespace ignite
*/
std::map<K, V> GetAll(const std::set<K>& keys, IgniteError& err)
{
+ std::map<K, V> res;
+
impl::InSetOperation<K> inOp(keys);
- impl::OutMapOperation<K, V> outOp;
+ impl::OutMapOperation<K, V> outOp(res);
impl.Get()->GetAll(inOp, outOp, err);
- return outOp.GetResult();
+ return res;
}
/**
@@ -515,12 +520,14 @@ namespace ignite
*/
V GetAndPut(const K& key, const V& val, IgniteError& err)
{
+ V oldVal;
+
impl::In2Operation<K, V> inOp(key, val);
- impl::Out1Operation<V> outOp;
+ impl::Out1Operation<V> outOp(oldVal);
impl.Get()->GetAndPut(inOp, outOp, err);
- return outOp.GetResult();
+ return oldVal;
}
/**
@@ -559,12 +566,14 @@ namespace ignite
*/
V GetAndReplace(const K& key, const V& val, IgniteError& err)
{
+ V oldVal;
+
impl::In2Operation<K, V> inOp(key, val);
- impl::Out1Operation<V> outOp;
+ impl::Out1Operation<V> outOp(oldVal);
impl.Get()->GetAndReplace(inOp, outOp, err);
- return outOp.GetResult();
+ return oldVal;
}
/**
@@ -597,12 +606,14 @@ namespace ignite
*/
V GetAndRemove(const K& key, IgniteError& err)
{
+ V oldVal;
+
impl::In1Operation<K> inOp(key);
- impl::Out1Operation<V> outOp;
+ impl::Out1Operation<V> outOp(oldVal);
impl.Get()->GetAndRemove(inOp, outOp, err);
- return outOp.GetResult();
+ return oldVal;
}
/**
@@ -694,12 +705,14 @@ namespace ignite
*/
V GetAndPutIfAbsent(const K& key, const V& val, IgniteError& err)
{
+ V oldVal;
+
impl::In2Operation<K, V> inOp(key, val);
- impl::Out1Operation<V> outOp;
+ impl::Out1Operation<V> outOp(oldVal);
impl.Get()->GetAndPutIfAbsent(inOp, outOp, err);
- return outOp.GetResult();
+ return oldVal;
}
/**
@@ -1607,14 +1620,15 @@ namespace ignite
{
typedef impl::cache::CacheEntryProcessorHolder<P, A> ProcessorHolder;
+ R res;
ProcessorHolder procHolder(processor, arg);
impl::In2Operation<K, ProcessorHolder> inOp(key, procHolder);
- impl::Out1Operation<R> outOp;
+ impl::Out1Operation<R> outOp(res);
impl.Get()->Invoke(inOp, outOp, err);
- return outOp.GetResult();
+ return res;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/40432b09/modules/platforms/cpp/core/include/ignite/cache/query/query_cursor.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/core/include/ignite/cache/query/query_cursor.h b/modules/platforms/cpp/core/include/ignite/cache/query/query_cursor.h
index 3f7ccce..e77706b 100644
--- a/modules/platforms/cpp/core/include/ignite/cache/query/query_cursor.h
+++ b/modules/platforms/cpp/core/include/ignite/cache/query/query_cursor.h
@@ -158,19 +158,14 @@ namespace ignite
impl::cache::query::QueryCursorImpl* impl0 = impl.Get();
if (impl0) {
- impl::Out2Operation<K, V> outOp;
+ K key;
+ V val;
- impl0->GetNext(outOp, err);
+ impl::Out2Operation<K, V> outOp(key, val);
- if (err.GetCode() == IgniteError::IGNITE_SUCCESS)
- {
- K& key = outOp.Get1();
- V& val = outOp.Get2();
+ impl0->GetNext(outOp, err);
- return CacheEntry<K, V>(key, val);
- }
- else
- return CacheEntry<K, V>();
+ return CacheEntry<K, V>(key, val);
}
else
{
http://git-wip-us.apache.org/repos/asf/ignite/blob/40432b09/modules/platforms/cpp/core/include/ignite/impl/operations.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/core/include/ignite/impl/operations.h b/modules/platforms/cpp/core/include/ignite/impl/operations.h
index 1645fb56..9f816bf 100644
--- a/modules/platforms/cpp/core/include/ignite/impl/operations.h
+++ b/modules/platforms/cpp/core/include/ignite/impl/operations.h
@@ -66,7 +66,7 @@ namespace ignite
public:
/**
* Constructor.
- *
+ *
* @param val Value.
*/
In1Operation(const T& val) : val(val)
@@ -248,7 +248,7 @@ namespace ignite
const T& key;
/** Peek modes. */
- int32_t peekModes;
+ int32_t peekModes;
IGNITE_NO_COPY_ASSIGNMENT(InCacheLocalPeekOperation)
};
@@ -333,15 +333,18 @@ namespace ignite
public:
/**
* Constructor.
+ *
+ * @param val Value.
*/
- Out1Operation()
+ Out1Operation(T& val) :
+ val(val)
{
// No-op.
}
virtual void ProcessOutput(binary::BinaryReaderImpl& reader)
{
- val = reader.ReadTopObject<T>();
+ reader.ReadTopObject<T>(val);
}
virtual void SetNull()
@@ -349,18 +352,9 @@ namespace ignite
val = binary::BinaryUtils::GetDefaultValue<T>();
}
- /**
- * Get value.
- *
- * @param Value.
- */
- T GetResult()
- {
- return val;
- }
private:
/** Value. */
- T val;
+ T& val;
IGNITE_NO_COPY_ASSIGNMENT(Out1Operation)
};
@@ -374,16 +368,21 @@ namespace ignite
public:
/**
* Constructor.
+ *
+ * @param val1 Value 1.
+ * @param val2 Value 2.
*/
- Out2Operation()
+ Out2Operation(T1& val1, T2& val2) :
+ val1(val1),
+ val2(val2)
{
// No-op.
}
virtual void ProcessOutput(binary::BinaryReaderImpl& reader)
{
- val1 = reader.ReadTopObject<T1>();
- val2 = reader.ReadTopObject<T2>();
+ reader.ReadTopObject<T1>(val1);
+ reader.ReadTopObject<T2>(val2);
}
virtual void SetNull()
@@ -392,32 +391,12 @@ namespace ignite
val2 = binary::BinaryUtils::GetDefaultValue<T2>();
}
- /**
- * Get value 1.
- *
- * @param Value 1.
- */
- T1& Get1()
- {
- return val1;
- }
-
- /**
- * Get value 2.
- *
- * @param Value 2.
- */
- T2& Get2()
- {
- return val2;
- }
-
private:
/** Value 1. */
- T1 val1;
-
+ T1& val1;
+
/** Value 2. */
- T2 val2;
+ T2& val2;
IGNITE_NO_COPY_ASSIGNMENT(Out2Operation)
};
@@ -431,18 +410,27 @@ namespace ignite
public:
/**
* Constructor.
+ *
+ * @param val1 Value 1.
+ * @param val2 Value 2.
+ * @param val3 Value 3.
+ * @param val4 Value 4.
*/
- Out4Operation()
+ Out4Operation(T1& val1, T2& val2, T3& val3, T4& val4) :
+ val1(val1),
+ val2(val2),
+ val3(val3),
+ val4(val4)
{
// No-op.
}
virtual void ProcessOutput(binary::BinaryReaderImpl& reader)
{
- val1 = reader.ReadTopObject<T1>();
- val2 = reader.ReadTopObject<T2>();
- val3 = reader.ReadTopObject<T3>();
- val4 = reader.ReadTopObject<T4>();
+ reader.ReadTopObject<T1>(val1);
+ reader.ReadTopObject<T2>(val2);
+ reader.ReadTopObject<T3>(val3);
+ reader.ReadTopObject<T4>(val4);
}
virtual void SetNull()
@@ -453,58 +441,18 @@ namespace ignite
val4 = binary::BinaryUtils::GetDefaultValue<T4>();
}
- /**
- * Get value 1.
- *
- * @param Value 1.
- */
- T1& Get1()
- {
- return val1;
- }
-
- /**
- * Get value 2.
- *
- * @param Value 2.
- */
- T2& Get2()
- {
- return val2;
- }
-
- /**
- * Get value 3.
- *
- * @param Value 3.
- */
- T3& Get3()
- {
- return val3;
- }
-
- /**
- * Get value 4.
- *
- * @param Value 4.
- */
- T4& Get4()
- {
- return val4;
- }
-
private:
/** Value 1. */
- T1 val1;
-
+ T1& val1;
+
/** Value 2. */
- T2 val2;
+ T2& val2;
/** Value 3. */
- T3 val3;
+ T3& val3;
/** Value 4. */
- T4 val4;
+ T4& val4;
IGNITE_NO_COPY_ASSIGNMENT(Out4Operation)
};
@@ -518,8 +466,11 @@ namespace ignite
public:
/**
* Constructor.
+ *
+ * @param val Value.
*/
- OutMapOperation()
+ OutMapOperation(std::map<T1, T2>& val) :
+ val(val)
{
// No-op.
}
@@ -550,18 +501,9 @@ namespace ignite
// No-op.
}
- /**
- * Get value.
- *
- * @return Value.
- */
- std::map<T1, T2> GetResult()
- {
- return val;
- }
private:
/** Value. */
- std::map<T1, T2> val;
+ std::map<T1, T2>& val;
IGNITE_NO_COPY_ASSIGNMENT(OutMapOperation)
};
@@ -587,7 +529,7 @@ namespace ignite
res.reserve(res.size() + cnt);
- for (int i = 0; i < cnt; i++)
+ for (int i = 0; i < cnt; i++)
{
K key = reader.ReadTopObject<K>();
V val = reader.ReadTopObject<V>();
@@ -604,7 +546,7 @@ namespace ignite
private:
/** Entries. */
std::vector<ignite::cache::CacheEntry<K, V> >& res;
-
+
IGNITE_NO_COPY_ASSIGNMENT(OutQueryGetAllOperation)
};
@@ -645,7 +587,7 @@ namespace ignite
private:
/** Out iter. */
Iter iter;
-
+
IGNITE_NO_COPY_ASSIGNMENT(OutQueryGetAllOperationIter)
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/40432b09/modules/platforms/cpp/odbc-test/src/connection_test.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/src/connection_test.cpp b/modules/platforms/cpp/odbc-test/src/connection_test.cpp
index 5badda6..709ef61 100644
--- a/modules/platforms/cpp/odbc-test/src/connection_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/connection_test.cpp
@@ -84,12 +84,15 @@ struct ConnectionTestSuiteFixture: odbc::OdbcTestSuite
/**
* Extract code from ODBC error message.
+ *
+ * @param err Error.
+ * @return Code.
*/
- std::string ExtractErrorCode(std::string err)
+ static std::string ExtractErrorCode(const std::string& err)
{
std::string code;
- int idx = err.find(':');
+ size_t idx = err.find(':');
if ((idx != std::string::npos) && (idx > 0))
code = err.substr(0, idx);
[26/50] [abbrv] ignite git commit: IGNITE-9360 Remove SnapTreeMap and
tests on it. - Fixes #4608.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/35706b94/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/package-info.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/package-info.java b/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/package-info.java
deleted file mode 100644
index 2d75a8c..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Snaptree and related classes.
- */
-package org.apache.ignite.internal.util.snaptree;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/35706b94/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index 4d15047..bf726d5 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -45,7 +45,6 @@ import org.apache.ignite.thread.GridThreadPoolExecutorServiceSelfTest;
import org.apache.ignite.thread.GridThreadTest;
import org.apache.ignite.thread.IgniteThreadPoolSizeTest;
import org.apache.ignite.util.GridConcurrentLinkedDequeMultiThreadedTest;
-import org.apache.ignite.util.GridIndexFillTest;
import org.apache.ignite.util.GridIntListSelfTest;
import org.apache.ignite.util.GridLogThrottleTest;
import org.apache.ignite.util.GridLongListSelfTest;
@@ -106,7 +105,6 @@ public class IgniteUtilSelfTestSuite extends TestSuite {
suite.addTestSuite(GridMessageCollectionTest.class);
suite.addTestSuite(WorkersControlMXBeanTest.class);
suite.addTestSuite(GridConcurrentLinkedDequeMultiThreadedTest.class);
- suite.addTestSuite(GridIndexFillTest.class);
suite.addTestSuite(GridLogThrottleTest.class);
suite.addTestSuite(GridRandomSelfTest.class);
suite.addTestSuite(GridSnapshotLockSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/35706b94/modules/core/src/test/java/org/apache/ignite/util/GridIndexFillTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridIndexFillTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridIndexFillTest.java
deleted file mode 100644
index 63635bd..0000000
--- a/modules/core/src/test/java/org/apache/ignite/util/GridIndexFillTest.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.util;
-
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.util.GridConcurrentSkipListSet;
-import org.apache.ignite.internal.util.snaptree.SnapTreeMap;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * TODO write doc
- */
-public class GridIndexFillTest extends GridCommonAbstractTest {
- /** */
- private CopyOnWriteArrayList<Idx> idxs;
-
- /** */
- private ConcurrentHashMap<Integer, CountDownLatch> keyLocks;
-
- @Override protected void beforeTest() throws Exception {
- super.beforeTest();
-
- idxs = new CopyOnWriteArrayList<>();
-
- idxs.add(new Idx(true));
-
- keyLocks = new ConcurrentHashMap<>();
- }
-
- /**
- * @param k Key.
- */
- private CountDownLatch lock(String op, Integer k) {
-// U.debug(op + " lock: " + k);
- CountDownLatch latch = new CountDownLatch(1);
-
- for(;;) {
- CountDownLatch l = keyLocks.putIfAbsent(k, latch);
-
- if (l == null)
- return latch;
-
- try {
- l.await();
- }
- catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- /**
- * @param k Key.
- */
- private void unlock(Integer k, CountDownLatch latch) {
-// U.debug("unlock: " + k);
- assertTrue(keyLocks.remove(k, latch));
-
- latch.countDown();
- }
-
- private void put(Integer k, Long v) {
- CountDownLatch l = lock("add", k);
-
- for (Idx idx : idxs)
- idx.add(k, v);
-
- unlock(k, l);
- }
-
- private void remove(Integer k) {
- CountDownLatch l = lock("rm", k);
-
- try {
- Long v = null;
-
- for (Idx idx : idxs) {
- Long v2 = idx.remove(k, v);
-
- if (v2 == null) {
- assert v == null;
-
- return; // Nothing to remove.
- }
-
- if (v == null)
- v = v2;
- else
- assert v.equals(v2);
- }
- }
- finally {
- unlock(k, l);
- }
- }
-
- public void testSnaptreeParallelBuild() throws Exception {
- final AtomicBoolean stop = new AtomicBoolean();
-
- IgniteInternalFuture<?> fut = multithreadedAsync(new Callable<Object>() {
- @Override public Object call() throws Exception {
- int i = ThreadLocalRandom.current().nextInt(100);
-
- while (!stop.get()) {
- int k = i++ % 100;
- long v = i++ % 10;
-
- if (i++ % 2 == 0)
- put(k, v);
- else
- remove(k);
- }
-
- return null;
- }
- }, 12, "put");
-
- Thread.sleep(500);
-
- Idx newIdx = new Idx(false);
-
- idxs.add(newIdx);
-
- SnapTreeMap<Integer, Long> snap = idxs.get(0).tree.clone();
-
- for (Map.Entry<Integer, Long> entry : snap.entrySet())
- newIdx.addX(entry.getKey(), entry.getValue());
-
- newIdx.finish();
-
- stop.set(true);
-
- fut.get();
-
- assertEquals(idxs.get(0).tree, idxs.get(1).tree);
- }
-
- private static class Idx {
-
- static int z = 1;
-
- private final SnapTreeMap<Integer, Long> tree = new SnapTreeMap<>(); //new ConcurrentSkipListMap<>();
-
- private volatile Rm rm;
-
- private final String name = "idx" + z++;
-
- public Idx(boolean pk) {
- if (!pk)
- rm = new Rm();
- }
-
- public void add(Integer k, Long v) {
-// U.debug(name + " add: k" + k + " " + v);
-
- Long old = tree.put(k, v);
-
- if (old != null) {
- Rm rm = this.rm;
-
- if (rm != null)
- rm.keys.add(k);
- }
- }
-
- public void addX(Integer k, Long v) {
-// U.debug(name + " addX: k" + k + " " + v);
-
- assert v != null;
- assert k != null;
-
-// Lock l = rm.lock.writeLock();
-
-// l.lock();
-
- try {
- if (!rm.keys.contains(k)) {
-// U.debug(name + " addX-put: k" + k + " " + v);
-
- tree.putIfAbsent(k, v);
- }
- }
- finally {
-// l.unlock();
- }
- }
-
- public Long remove(Integer k, Long v) {
- Rm rm = this.rm;
-
- if (rm != null) {
- assert v != null;
-
-// Lock l = rm.lock.readLock();
-
-// l.lock();
-
- try {
- rm.keys.add(k);
-
- Long v2 = tree.remove(k);
-
-// U.debug(name + " rm1: k" + k + " " + v + " " + v2);
-
- }
- finally {
-// l.unlock();
- }
- }
- else {
- Long v2 = tree.remove(k);
-
-// U.debug(name + " rm2: k" + k + " " + v + " " + v2);
-
- if (v == null)
- v = v2;
- else
- assertEquals(v, v2);
- }
-
- return v;
- }
-
- public void finish() {
-// assertTrue(rm.tree.isEmpty());
-
- rm = null;
- }
- }
-
- private static class Rm {
-// private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-
- private final GridConcurrentSkipListSet<Integer> keys = new GridConcurrentSkipListSet<>();
- //new SnapTreeMap<>(); //new ConcurrentSkipListMap<>();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/35706b94/modules/indexing/src/test/java/org/apache/ignite/loadtests/h2indexing/GridTestEntity.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/loadtests/h2indexing/GridTestEntity.java b/modules/indexing/src/test/java/org/apache/ignite/loadtests/h2indexing/GridTestEntity.java
deleted file mode 100644
index 015ec96..0000000
--- a/modules/indexing/src/test/java/org/apache/ignite/loadtests/h2indexing/GridTestEntity.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.loadtests.h2indexing;
-
-import java.util.Date;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-
-/**
- * Test entity.
- */
-public class GridTestEntity {
- /** */
- @QuerySqlField(index = true)
- private final String name;
-
- /** */
- @QuerySqlField(index = false)
- private final Date date;
-
- /**
- * Constructor.
- *
- * @param name Name.
- * @param date Date.
- */
- @SuppressWarnings("AssignmentToDateFieldFromParameter")
- public GridTestEntity(String name, Date date) {
- this.name = name;
- this.date = date;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- GridTestEntity that = (GridTestEntity) o;
-
- return !(date != null ? !date.equals(that.date) : that.date != null) &&
- !(name != null ? !name.equals(that.name) : that.name != null);
-
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = name != null ? name.hashCode() : 0;
-
- res = 31 * res + (date != null ? date.hashCode() : 0);
-
- return res;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/35706b94/modules/indexing/src/test/java/org/apache/ignite/loadtests/h2indexing/GridTreeBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/loadtests/h2indexing/GridTreeBenchmark.java b/modules/indexing/src/test/java/org/apache/ignite/loadtests/h2indexing/GridTreeBenchmark.java
deleted file mode 100644
index a711468..0000000
--- a/modules/indexing/src/test/java/org/apache/ignite/loadtests/h2indexing/GridTreeBenchmark.java
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.loadtests.h2indexing;
-
-import java.util.Iterator;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.ConcurrentNavigableMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.ignite.internal.util.snaptree.SnapTreeMap;
-
-/**
- * NavigableMaps PUT benchmark.
- */
-public class GridTreeBenchmark {
- /** */
- private static final int PUTS = 8000000;
-
- /** */
- private static final int THREADS = 8;
-
- /** */
- private static final int ITERATIONS = PUTS / THREADS;
-
- /**
- * Main method.
- *
- * @param args Command line args (not used).
- * @throws BrokenBarrierException If failed.
- * @throws InterruptedException If failed.
- */
- public static void main(String... args) throws BrokenBarrierException, InterruptedException {
- doTestMaps();
- }
-
- /**
- * @throws BrokenBarrierException If failed.
- * @throws InterruptedException If failed.
- */
- private static void doTestAtomicInt() throws BrokenBarrierException, InterruptedException {
- final AtomicInteger[] cnts = new AtomicInteger[8];
-
- for (int i = 0; i < cnts.length; i++)
- cnts[i] = new AtomicInteger();
-
- final Thread[] ths = new Thread[THREADS];
-
- final CyclicBarrier barrier = new CyclicBarrier(THREADS + 1);
-
- final AtomicInteger cnt = new AtomicInteger();
-
- for (int i = 0; i < ths.length; i++) {
- ths[i] = new Thread(new Runnable() {
- @Override public void run() {
- int idx = cnt.getAndIncrement();
-
- AtomicInteger x = cnts[idx % cnts.length];
-
- try {
- barrier.await();
- }
- catch (Exception e) {
- throw new IllegalStateException(e);
- }
-
- for (int i = 0; i < ITERATIONS; i++)
- x.incrementAndGet();
- }
- });
-
- ths[i].start();
- }
-
- barrier.await();
-
- long start = System.currentTimeMillis();
-
- for (Thread t : ths)
- t.join();
-
- long time = System.currentTimeMillis() - start;
-
- System.out.println(time);
-
- }
-
- /**
- * @throws BrokenBarrierException If failed.
- * @throws InterruptedException If failed.
- */
- private static void doTestMaps() throws BrokenBarrierException, InterruptedException {
- final UUID[] data = generate();
-
- @SuppressWarnings("unchecked")
- final Map<UUID, UUID>[] maps = new Map[4];
-
- for (int i = 0; i < maps.length; i++)
- maps[i] =
- new SnapTreeMap<>();
-
-
- final Thread[] ths = new Thread[THREADS];
-
- final CyclicBarrier barrier = new CyclicBarrier(THREADS + 1);
-
- final AtomicInteger cnt = new AtomicInteger();
-
- for (int i = 0; i < ths.length; i++) {
- ths[i] = new Thread(new Runnable() {
- @Override public void run() {
- int idx = cnt.getAndIncrement();
-
- int off = idx * ITERATIONS;
-
- Map<UUID, UUID> map = maps[idx % maps.length];
-
- try {
- barrier.await();
- }
- catch (Exception e) {
- throw new IllegalStateException(e);
- }
-
- for (int i = 0; i < ITERATIONS; i++) {
- UUID id = data[off + i];
-
- id = map.put(id, id);
-
- assert id == null;
- }
- }
- });
-
- ths[i].start();
- }
-
- System.out.println("Sleep");
- Thread.sleep(10000);
-
- System.out.println("Go");
- barrier.await();
-
- long start = System.currentTimeMillis();
-
- for (Thread t : ths)
- t.join();
-
- long time = System.currentTimeMillis() - start;
-
- System.out.println(time);
- }
-
- /**
- * @throws BrokenBarrierException If failed.
- * @throws InterruptedException If failed.
- */
- private static void doBenchmark() throws BrokenBarrierException, InterruptedException {
- int attemts = 20;
- int warmups = 10;
-
- long snapTreeTime = 0;
- long skipListTime = 0;
-
- for (int i = 0; i < attemts; i++) {
- ConcurrentNavigableMap<UUID, UUID> skipList = new ConcurrentSkipListMap<>();
- ConcurrentNavigableMap<UUID, UUID> snapTree = new SnapTreeMap<>();
-
- UUID[] ids = generate();
-
- boolean warmup = i < warmups;
-
- snapTreeTime += doTest(snapTree, ids, warmup);
- skipListTime += doTest(skipList, ids, warmup);
-
- assert skipList.size() == snapTree.size();
-
- Iterator<UUID> snapIt = snapTree.keySet().iterator();
- Iterator<UUID> listIt = skipList.keySet().iterator();
-
- for (int x = 0, len = skipList.size(); x < len; x++)
- assert snapIt.next() == listIt.next();
-
- System.out.println(i + " ==================");
- }
-
- attemts -= warmups;
-
- System.out.println("Avg for GridSnapTreeMap: " + (snapTreeTime / attemts) + " ms");
- System.out.println("Avg for ConcurrentSkipListMap: " + (skipListTime / attemts) + " ms");
- }
-
- /**
- * @return UUIDs.
- */
- private static UUID[] generate() {
- UUID[] ids = new UUID[ITERATIONS * THREADS];
-
- for (int i = 0; i < ids.length; i++)
- ids[i] = UUID.randomUUID();
-
- return ids;
- }
-
- /**
- * @param tree Tree.
- * @param data Data.
- * @param warmup Warmup.
- * @return Time.
- * @throws BrokenBarrierException If failed.
- * @throws InterruptedException If failed.
- */
- private static long doTest(final ConcurrentNavigableMap<UUID, UUID> tree, final UUID[] data, boolean warmup)
- throws BrokenBarrierException, InterruptedException {
- Thread[] ths = new Thread[THREADS];
-
- final CyclicBarrier barrier = new CyclicBarrier(THREADS + 1);
-
- final AtomicInteger cnt = new AtomicInteger();
-
- for (int i = 0; i < ths.length; i++) {
- ths[i] = new Thread(new Runnable() {
- @Override public void run() {
- int off = cnt.getAndIncrement() * ITERATIONS;
-
- try {
- barrier.await();
- }
- catch (Exception e) {
- throw new IllegalStateException(e);
- }
-
- for (int i = 0; i < ITERATIONS; i++) {
- UUID id = data[off + i];
-
- id = tree.put(id, id);
-
- assert id == null;
- }
- }
- });
-
- ths[i].start();
- }
-
- barrier.await();
-
- long start = System.currentTimeMillis();
-
- for (Thread t : ths)
- t.join();
-
- long time = System.currentTimeMillis() - start;
-
- if (!warmup) {
- System.out.println(tree.getClass().getSimpleName() + " " + time + " ms");
-
- return time;
- }
-
- return 0;
- }
-}
[27/50] [abbrv] ignite git commit: IGNITE-9360 Remove SnapTreeMap and
tests on it. - Fixes #4608.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/35706b94/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/SnapTreeMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/SnapTreeMap.java b/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/SnapTreeMap.java
deleted file mode 100644
index dce2fb8..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/snaptree/SnapTreeMap.java
+++ /dev/null
@@ -1,2917 +0,0 @@
-/*
- * Copyright (c) 2009 Stanford University, unless otherwise specified.
- * All rights reserved.
- *
- * This software was developed by the Pervasive Parallelism Laboratory of
- * Stanford University, California, USA.
- *
- * Permission to use, copy, modify, and distribute this software in source
- * or binary form for any purpose with or without fee is hereby granted,
- * provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * 3. Neither the name of Stanford University nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-package org.apache.ignite.internal.util.snaptree;
-
-import org.apache.ignite.*;
-import org.apache.ignite.internal.util.*;
-import org.apache.ignite.internal.util.lang.*;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.util.AbstractMap;
-import java.util.AbstractSet;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.NoSuchElementException;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.concurrent.ConcurrentNavigableMap;
-
-// TODO: optimized buildFromSorted
-// TODO: submap.clone()
-
-/** A concurrent AVL tree with fast cloning, based on the algorithm of Bronson,
- * Casper, Chafi, and Olukotun, "A Practical Concurrent Binary Search Tree"
- * published in PPoPP'10. To simplify the locking protocols rebalancing work
- * is performed in pieces, and some removed keys are be retained as routing
- * nodes in the tree.
- *
- * <p>This data structure honors all of the contracts of {@link
- * java.util.concurrent.ConcurrentSkipListMap}, with the additional contract
- * that clone, size, toArray, and iteration are linearizable (atomic).
- *
- * <p>The tree uses optimistic concurrency control. No locks are usually
- * required for get, containsKey, firstKey, firstEntry, lastKey, or lastEntry.
- * Reads are not lock free (or even obstruction free), but obstructing threads
- * perform no memory allocation, system calls, or loops, which seems to work
- * okay in practice. All of the updates to the tree are performed in fixed-
- * size blocks, so restoration of the AVL balance criteria may occur after a
- * change to the tree has linearized (but before the mutating operation has
- * returned). The tree is always properly balanced when quiescent.
- *
- * <p>To clone the tree (or produce a snapshot for consistent iteration) the
- * root node is marked as shared, which must be (*) done while there are no
- * pending mutations. New mutating operations are blocked if a mark is
- * pending, and when existing mutating operations are completed the mark is
- * made.
- * <em>* - It would be less disruptive if we immediately marked the root as
- * shared, and then waited for pending operations that might not have seen the
- * mark without blocking new mutations. This could result in imbalance being
- * frozen into the shared portion of the tree, though. To minimize the
- * problem we perform the mark and reenable mutation on whichever thread
- * notices that the entry count has become zero, to reduce context switches on
- * the critical path.</em>
- *
- * <p>The same multi-cache line data structure required for efficiently
- * tracking the entry and exit for mutating operations is used to maintain the
- * current size of the tree. This means that the size can be computed by
- * quiescing as for a clone, but without doing any marking.
- *
- * <p>Range queries such as higherKey are not amenable to the optimistic
- * hand-over-hand locking scheme used for exact searches, so they are
- * implemented with pessimistic concurrency control. Mutation can be
- * considered to acquire a lock on the map in Intention-eXclusive mode, range
- * queries, size(), and root marking acquire the lock in Shared mode.
- *
- * @author Nathan Bronson
- */
-@SuppressWarnings("ALL")
-public class SnapTreeMap<K, V> extends AbstractMap<K, V> implements ConcurrentNavigableMap<K, V>, Cloneable,
- Serializable {
-
- /** */
- private static final long serialVersionUID = 0L;
-
- /** If false, null values will trigger a NullPointerException. When false,
- * this map acts exactly like a ConcurrentSkipListMap, except for the
- * running time of the methods. The ability to get a snapshot reduces the
- * potential ambiguity between null values and absent entries, so I'm not
- * sure what the default should be.
- */
- static final boolean AllowNullValues = false;
-
- /** This is a special value that indicates the presence of a null value,
- * to differentiate from the absence of a value. Only used when
- * {@link #AllowNullValues} is true.
- */
- static final Object SpecialNull = new Object();
-
- /** This is a special value that indicates that an optimistic read
- * failed.
- */
- static final Object SpecialRetry = new Object();
-
-
- /** The number of spins before yielding. */
- static final int SpinCount = Integer.parseInt(System.getProperty("snaptree.spin", "100"));
-
- /** The number of yields before blocking. */
- static final int YieldCount = Integer.parseInt(System.getProperty("snaptree.yield", "0"));
-
-
- // we encode directions as characters
- static final char Left = 'L';
- static final char Right = 'R';
-
-
- /** An <tt>OVL</tt> is a version number and lock used for optimistic
- * concurrent control of some program invariant. If {@link #isShrinking}
- * then the protected invariant is changing. If two reads of an OVL are
- * performed that both see the same non-changing value, the reader may
- * conclude that no changes to the protected invariant occurred between
- * the two reads. The special value UnlinkedOVL is not changing, and is
- * guaranteed to not result from a normal sequence of beginChange and
- * endChange operations.
- * <p>
- * For convenience <tt>endChange(ovl) == endChange(beginChange(ovl))</tt>.
- */
- static long beginChange(long ovl) { return ovl | 1; }
- static long endChange(long ovl) { return (ovl | 3) + 1; }
- static final long UnlinkedOVL = 2;
-
- static boolean isShrinking(long ovl) { return (ovl & 1) != 0; }
- static boolean isUnlinked(long ovl) { return (ovl & 2) != 0; }
- static boolean isShrinkingOrUnlinked(long ovl) { return (ovl & 3) != 0L; }
-
-
- protected static class Node<K,V> implements Map.Entry<K,V> {
- public K key;
- volatile int height;
-
- /** null means this node is conceptually not present in the map.
- * SpecialNull means the value is null.
- */
- volatile Object vOpt;
- volatile Node<K,V> parent;
- volatile long shrinkOVL;
- volatile Node<K,V> left;
- volatile Node<K,V> right;
-
- Node(final K key,
- final int height,
- final Object vOpt,
- final Node<K,V> parent,
- final long shrinkOVL,
- final Node<K,V> left,
- final Node<K,V> right)
- {
- this.key = key;
- this.height = height;
- this.vOpt = vOpt;
- this.parent = parent;
- this.shrinkOVL = shrinkOVL;
- this.left = left;
- this.right = right;
- }
-
- @Override
- public K getKey() { return key; }
-
- @Override
- @SuppressWarnings("unchecked")
- public V getValue() {
- final Object tmp = vOpt;
- if (AllowNullValues) {
- return tmp == SpecialNull ? null : (V)tmp;
- } else {
- return (V)tmp;
- }
- }
-
- @Override
- public V setValue(final V v) {
- throw new UnsupportedOperationException();
- }
-
- Node<K,V> child(char dir) { return dir == Left ? left : right; }
-
- void setChild(char dir, Node<K,V> node) {
- if (dir == Left) {
- left = node;
- } else {
- right = node;
- }
- }
-
- //////// copy-on-write stuff
-
- private static <K,V> boolean isShared(final Node<K,V> node) {
- return node != null && node.parent == null;
- }
-
- static <K,V> Node<K,V> markShared(final Node<K,V> node) {
- if (node != null) {
- node.parent = null;
- }
- return node;
- }
-
- private Node<K,V> lazyCopy(Node<K,V> newParent) {
- assert (isShared(this));
- assert (!isShrinkingOrUnlinked(shrinkOVL));
-
- return new Node<K,V>(key, height, vOpt, newParent, 0L, markShared(left), markShared(right));
- }
-
- Node<K,V> unsharedLeft() {
- final Node<K,V> cl = left;
- if (!isShared(cl)) {
- return cl;
- } else {
- lazyCopyChildren();
- return left;
- }
- }
-
- Node<K,V> unsharedRight() {
- final Node<K,V> cr = right;
- if (!isShared(cr)) {
- return cr;
- } else {
- lazyCopyChildren();
- return right;
- }
- }
-
- Node<K,V> unsharedChild(final char dir) {
- return dir == Left ? unsharedLeft() : unsharedRight();
- }
-
- private synchronized void lazyCopyChildren() {
- final Node<K,V> cl = left;
- if (isShared(cl)) {
- left = cl.lazyCopy(this);
- }
- final Node<K,V> cr = right;
- if (isShared(cr)) {
- right = cr.lazyCopy(this);
- }
- }
-
- //////// per-node blocking
-
- private void waitUntilShrinkCompleted(final long ovl) {
- if (!isShrinking(ovl)) {
- return;
- }
-
- for (int tries = 0; tries < SpinCount; ++tries) {
- if (shrinkOVL != ovl) {
- return;
- }
- }
-
- for (int tries = 0; tries < YieldCount; ++tries) {
- Thread.yield();
- if (shrinkOVL != ovl) {
- return;
- }
- }
-
- // spin and yield failed, use the nuclear option
- synchronized (this) {
- // we can't have gotten the lock unless the shrink was over
- }
- assert(shrinkOVL != ovl);
- }
-
- int validatedHeight() {
- final int hL = left == null ? 0 : left.validatedHeight();
- final int hR = right == null ? 0 : right.validatedHeight();
- assert(Math.abs(hL - hR) <= 1);
- final int h = 1 + Math.max(hL, hR);
- assert(h == height);
- return height;
- }
-
- //////// SubMap.size() helper
-
- static <K,V> int computeFrozenSize(Node<K,V> root,
- Comparable<? super K> fromCmp,
- boolean fromIncl,
- final Comparable<? super K> toCmp,
- final boolean toIncl) {
- int result = 0;
- while (true) {
- if (root == null) {
- return result;
- }
- if (fromCmp != null) {
- final int c = fromCmp.compareTo(root.key);
- if (c > 0 || (c == 0 && !fromIncl)) {
- // all matching nodes are on the right side
- root = root.right;
- continue;
- }
- }
- if (toCmp != null) {
- final int c = toCmp.compareTo(root.key);
- if (c < 0 || (c == 0 && !toIncl)) {
- // all matching nodes are on the left side
- root = root.left;
- continue;
- }
- }
-
- // Current node matches. Nodes on left no longer need toCmp, nodes
- // on right no longer need fromCmp.
- if (root.vOpt != null) {
- ++result;
- }
- result += computeFrozenSize(root.left, fromCmp, fromIncl, null, false);
- fromCmp = null;
- root = root.right;
- }
- }
-
- //////// Map.Entry stuff
-
- @Override
- public boolean equals(final Object o) {
- if (!(o instanceof Map.Entry)) {
- return false;
- }
- final Map.Entry rhs = (Map.Entry)o;
- return eq(key, rhs.getKey()) && eq(getValue(), rhs.getValue());
- }
-
- private static boolean eq(final Object o1, final Object o2) {
- return o1 == null ? o2 == null : o1.equals(o2);
- }
-
- @Override
- public int hashCode() {
- return (key == null ? 0 : key.hashCode()) ^
- (getValue() == null ? 0 : getValue().hashCode());
- }
-
- @Override
- public String toString() {
- return key + "=" + getValue();
- }
- }
-
- private static class RootHolder<K,V> extends Node<K,V> {
- RootHolder() {
- super(null, 1, null, null, 0L, null, null);
- }
-
- RootHolder(final RootHolder<K,V> snapshot) {
- super(null, 1 + snapshot.height, null, null, 0L, null, snapshot.right);
- }
- }
-
- private static class COWMgr<K,V> extends CopyOnWriteManager<RootHolder<K,V>> {
- COWMgr() {
- super(new RootHolder<K,V>(), 0);
- }
-
- COWMgr(final RootHolder<K,V> initialValue, final int initialSize) {
- super(initialValue, initialSize);
- }
-
- protected RootHolder<K,V> freezeAndClone(final RootHolder<K,V> value) {
- Node.markShared(value.right);
- return new RootHolder<K,V>(value);
- }
-
- protected RootHolder<K,V> cloneFrozen(final RootHolder<K,V> frozenValue) {
- return new RootHolder<K,V>(frozenValue);
- }
- }
-
- //////// node access functions
-
- private static int height(final Node<?,?> node) {
- return node == null ? 0 : node.height;
- }
-
- @SuppressWarnings("unchecked")
- private V decodeNull(final Object vOpt) {
- assert (vOpt != SpecialRetry);
- if (AllowNullValues) {
- return vOpt == SpecialNull ? null : (V)vOpt;
- } else {
- return (V)vOpt;
- }
- }
-
- private static Object encodeNull(final Object v) {
- if (AllowNullValues) {
- return v == null ? SpecialNull : v;
- } else {
- if (v == null) {
- throw new NullPointerException();
- }
- return v;
- }
- }
-
- //////////////// state
-
- private final Comparator<? super K> comparator;
- private transient volatile COWMgr<K,V> holderRef;
-
- //////////////// public interface
-
- public SnapTreeMap() {
- this.comparator = null;
- this.holderRef = new COWMgr<K,V>();
- }
-
- public SnapTreeMap(final Comparator<? super K> comparator) {
- this.comparator = comparator;
- this.holderRef = new COWMgr<K,V>();
- }
-
- public SnapTreeMap(final Map<? extends K, ? extends V> source) {
- this.comparator = null;
- this.holderRef = new COWMgr<K,V>();
- putAll(source);
- }
-
- public SnapTreeMap(final SortedMap<K,? extends V> source) {
- this.comparator = source.comparator();
- if (source instanceof SnapTreeMap) {
- final SnapTreeMap<K,V> s = (SnapTreeMap<K,V>) source;
- this.holderRef = (COWMgr<K,V>) s.holderRef.clone();
- }
- else {
- // TODO: take advantage of the sort order
- // for now we optimize only by bypassing the COWMgr
- int size = 0;
- final RootHolder<K,V> holder = new RootHolder<K,V>();
- for (Map.Entry<K,? extends V> e : source.entrySet()) {
- final K k = e.getKey();
- final V v = e.getValue();
- if (k == null) {
- throw new NullPointerException("source map contained a null key");
- }
- if (!AllowNullValues && v == null) {
- throw new NullPointerException("source map contained a null value");
- }
- updateUnderRoot(k, comparable(k), UpdateAlways, null, encodeNull(v), holder);
- ++size;
- }
-
- this.holderRef = new COWMgr<K,V>(holder, size);
- }
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public SnapTreeMap<K,V> clone() {
- final SnapTreeMap<K,V> copy;
- try {
- copy = (SnapTreeMap<K,V>) super.clone();
- } catch (final CloneNotSupportedException xx) {
- throw new InternalError();
- }
- assert(copy.comparator == comparator);
- copy.holderRef = (COWMgr<K,V>) holderRef.clone();
- return copy;
- }
-
- @Override
- public int size() {
- return holderRef.size();
- }
-
- @Override
- public boolean isEmpty() {
- // removed-but-not-unlinked nodes cannot be leaves, so if the tree is
- // truly empty then the root holder has no right child
- return holderRef.read().right == null;
- }
-
- @Override
- public void clear() {
- holderRef = new COWMgr<K,V>();
- }
-
- @Override
- public Comparator<? super K> comparator() {
- return comparator;
- }
-
- @Override
- public boolean containsValue(final Object value) {
- // apply the same null policy as the rest of the code, but fall
- // back to the default implementation
- encodeNull(value);
- return super.containsValue(value);
- }
-
- //////// concurrent search
-
- @Override
- public boolean containsKey(final Object key) {
- return getImpl(key) != null;
- }
-
- @Override
- public V get(final Object key) {
- return decodeNull(getImpl(key));
- }
-
- @SuppressWarnings("unchecked")
- protected Comparable<? super K> comparable(final Object key) {
- if (key == null) {
- throw new NullPointerException();
- }
- if (comparator == null) {
- return (Comparable<? super K>)key;
- }
- return new Comparable<K>() {
- final Comparator<? super K> _cmp = comparator;
-
- @SuppressWarnings("unchecked")
- public int compareTo(final K rhs) { return _cmp.compare((K)key, rhs); }
- };
- }
-
- /** Returns either a value or SpecialNull, if present, or null, if absent. */
- private Object getImpl(final Object key) {
- final Comparable<? super K> k = comparable(key);
-
- while (true) {
- final Node<K,V> right = holderRef.read().right;
- if (right == null) {
- return null;
- } else {
- final int rightCmp = k.compareTo(right.key);
- if (rightCmp == 0) {
- // who cares how we got here
- return right.vOpt;
- }
-
- final long ovl = right.shrinkOVL;
- if (isShrinkingOrUnlinked(ovl)) {
- right.waitUntilShrinkCompleted(ovl);
- // RETRY
- } else if (right == holderRef.read().right) {
- // the reread of .right is the one protected by our read of ovl
- final Object vo = attemptGet(k, right, (rightCmp < 0 ? Left : Right), ovl);
- if (vo != SpecialRetry) {
- return vo;
- }
- // else RETRY
- }
- }
- }
- }
-
- private Object attemptGet(final Comparable<? super K> k,
- final Node<K,V> node,
- final char dirToC,
- final long nodeOVL) {
- while (true) {
- final Node<K,V> child = node.child(dirToC);
-
- if (child == null) {
- if (node.shrinkOVL != nodeOVL) {
- return SpecialRetry;
- }
-
- // Note is not present. Read of node.child occurred while
- // parent.child was valid, so we were not affected by any
- // shrinks.
- return null;
- } else {
- final int childCmp = k.compareTo(child.key);
- if (childCmp == 0) {
- // how we got here is irrelevant
- return child.vOpt;
- }
-
- // child is non-null
- final long childOVL = child.shrinkOVL;
- if (isShrinkingOrUnlinked(childOVL)) {
- child.waitUntilShrinkCompleted(childOVL);
-
- if (node.shrinkOVL != nodeOVL) {
- return SpecialRetry;
- }
- // else RETRY
- } else if (child != node.child(dirToC)) {
- // this .child is the one that is protected by childOVL
- if (node.shrinkOVL != nodeOVL) {
- return SpecialRetry;
- }
- // else RETRY
- } else {
- if (node.shrinkOVL != nodeOVL) {
- return SpecialRetry;
- }
-
- // At this point we know that the traversal our parent took
- // to get to node is still valid. The recursive
- // implementation will validate the traversal from node to
- // child, so just prior to the nodeOVL validation both
- // traversals were definitely okay. This means that we are
- // no longer vulnerable to node shrinks, and we don't need
- // to validate nodeOVL any more.
- final Object vo = attemptGet(k, child, (childCmp < 0 ? Left : Right), childOVL);
- if (vo != SpecialRetry) {
- return vo;
- }
- // else RETRY
- }
- }
- }
- }
-
- @Override
- public K firstKey() {
- return extremeKeyOrThrow(Left);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Map.Entry<K,V> firstEntry() {
- return (SimpleImmutableEntry<K,V>) extreme(false, Left);
- }
-
- @Override
- public K lastKey() {
- return extremeKeyOrThrow(Right);
- }
-
- @SuppressWarnings("unchecked")
- public Map.Entry<K,V> lastEntry() {
- return (SimpleImmutableEntry<K,V>) extreme(false, Right);
- }
-
- private K extremeKeyOrThrow(final char dir) {
- final K k = (K) extreme(true, dir);
- if (k == null) {
- throw new NoSuchElementException();
- }
- return k;
- }
-
- /** Returns a key if returnKey is true, a SimpleImmutableEntry otherwise.
- * Returns null if none exists.
- */
- private Object extreme(final boolean returnKey, final char dir) {
- while (true) {
- final Node<K,V> right = holderRef.read().right;
- if (right == null) {
- return null;
- } else {
- final long ovl = right.shrinkOVL;
- if (isShrinkingOrUnlinked(ovl)) {
- right.waitUntilShrinkCompleted(ovl);
- // RETRY
- } else if (right == holderRef.read().right) {
- // the reread of .right is the one protected by our read of ovl
- final Object vo = attemptExtreme(returnKey, dir, right, ovl);
- if (vo != SpecialRetry) {
- return vo;
- }
- // else RETRY
- }
- }
- }
- }
-
- private Object attemptExtreme(final boolean returnKey,
- final char dir,
- final Node<K,V> node,
- final long nodeOVL) {
- while (true) {
- final Node<K,V> child = node.child(dir);
-
- if (child == null) {
- // read of the value must be protected by the OVL, because we
- // must linearize against another thread that inserts a new min
- // key and then changes this key's value
- final Object vo = node.vOpt;
-
- if (node.shrinkOVL != nodeOVL) {
- return SpecialRetry;
- }
-
- assert(vo != null);
-
- return returnKey ? node.key : new SimpleImmutableEntry<K,V>(node.key, decodeNull(vo));
- } else {
- // child is non-null
- final long childOVL = child.shrinkOVL;
- if (isShrinkingOrUnlinked(childOVL)) {
- child.waitUntilShrinkCompleted(childOVL);
-
- if (node.shrinkOVL != nodeOVL) {
- return SpecialRetry;
- }
- // else RETRY
- } else if (child != node.child(dir)) {
- // this .child is the one that is protected by childOVL
- if (node.shrinkOVL != nodeOVL) {
- return SpecialRetry;
- }
- // else RETRY
- } else {
- if (node.shrinkOVL != nodeOVL) {
- return SpecialRetry;
- }
-
- final Object vo = attemptExtreme(returnKey, dir, child, childOVL);
- if (vo != SpecialRetry) {
- return vo;
- }
- // else RETRY
- }
- }
- }
- }
-
- //////////////// quiesced search
-
- @Override
- @SuppressWarnings("unchecked")
- public K lowerKey(final K key) {
- return (K) boundedExtreme(null, false, comparable(key), false, true, Right);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public K floorKey(final K key) {
- return (K) boundedExtreme(null, false, comparable(key), true, true, Right);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public K ceilingKey(final K key) {
- return (K) boundedExtreme(comparable(key), true, null, false, true, Left);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public K higherKey(final K key) {
- return (K) boundedExtreme(comparable(key), false, null, false, true, Left);
- }
-
-
- @Override
- @SuppressWarnings("unchecked")
- public Entry<K,V> lowerEntry(final K key) {
- return (Entry<K,V>) boundedExtreme(null, false, comparable(key), false, false, Right);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Entry<K,V> floorEntry(final K key) {
- return (Entry<K,V>) boundedExtreme(null, false, comparable(key), true, false, Right);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Entry<K,V> ceilingEntry(final K key) {
- return (Entry<K,V>) boundedExtreme(comparable(key), true, null, false, false, Left);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Entry<K,V> higherEntry(final K key) {
- return (Entry<K,V>) boundedExtreme(comparable(key), false, null, false, false, Left);
- }
-
- /** Returns null if none exists. */
- @SuppressWarnings("unchecked")
- private K boundedExtremeKeyOrThrow(final Comparable<? super K> minCmp,
- final boolean minIncl,
- final Comparable<? super K> maxCmp,
- final boolean maxIncl,
- final char dir) {
- final K k = (K) boundedExtreme(minCmp, minIncl, maxCmp, maxIncl, true, dir);
- if (k == null) {
- throw new NoSuchElementException();
- }
- return k;
- }
-
- /** Returns null if none exists. */
- @SuppressWarnings("unchecked")
- private Object boundedExtreme(final Comparable<? super K> minCmp,
- final boolean minIncl,
- final Comparable<? super K> maxCmp,
- final boolean maxIncl,
- final boolean returnKey,
- final char dir) {
- K resultKey;
- Object result;
-
- if ((dir == Left && minCmp == null) || (dir == Right && maxCmp == null)) {
- // no bound in the extreme direction, so use the concurrent search
- result = extreme(returnKey, dir);
- if (result == null) {
- return null;
- }
- resultKey = returnKey ? (K) result : ((SimpleImmutableEntry<K,V>) result).getKey();
- }
- else {
- RootHolder holder = holderRef.availableFrozen();
- final Epoch.Ticket ticket;
- if (holder == null) {
- ticket = holderRef.beginQuiescent();
- holder = holderRef.read();
- }
- else {
- ticket = null;
- }
- try {
- final Node<K,V> node = (dir == Left)
- ? boundedMin(holder.right, minCmp, minIncl)
- : boundedMax(holder.right, maxCmp, maxIncl);
- if (node == null) {
- return null;
- }
- resultKey = node.key;
- if (returnKey) {
- result = node.key;
- }
- else if (ticket == null) {
- // node of a frozen tree is okay, copy otherwise
- result = node;
- }
- else {
- // we must copy the node
- result = new SimpleImmutableEntry<K,V>(node.key, node.getValue());
- }
- }
- finally {
- if (ticket != null) {
- ticket.leave(0);
- }
- }
- }
-
- if (dir == Left && maxCmp != null) {
- final int c = maxCmp.compareTo(resultKey);
- if (c < 0 || (c == 0 && !maxIncl)) {
- return null;
- }
- }
- if (dir == Right && minCmp != null) {
- final int c = minCmp.compareTo(resultKey);
- if (c > 0 || (c == 0 && !minIncl)) {
- return null;
- }
- }
-
- return result;
- }
-
- private Node<K,V> boundedMin(Node<K,V> node,
- final Comparable<? super K> minCmp,
- final boolean minIncl) {
- while (node != null) {
- final int c = minCmp.compareTo(node.key);
- if (c < 0) {
- // there may be a matching node on the left branch
- final Node<K,V> z = boundedMin(node.left, minCmp, minIncl);
- if (z != null) {
- return z;
- }
- }
-
- if (c < 0 || (c == 0 && minIncl)) {
- // this node is a candidate, is it actually present?
- if (node.vOpt != null) {
- return node;
- }
- }
-
- // the matching node is on the right branch if it is present
- node = node.right;
- }
- return null;
- }
-
- private Node<K,V> boundedMax(Node<K,V> node,
- final Comparable<? super K> maxCmp,
- final boolean maxIncl) {
- while (node != null) {
- final int c = maxCmp.compareTo(node.key);
- if (c > 0) {
- // there may be a matching node on the right branch
- final Node<K,V> z = boundedMax(node.right, maxCmp, maxIncl);
- if (z != null) {
- return z;
- }
- }
-
- if (c > 0 || (c == 0 && maxIncl)) {
- // this node is a candidate, is it actually present?
- if (node.vOpt != null) {
- return node;
- }
- }
-
- // the matching node is on the left branch if it is present
- node = node.left;
- }
- return null;
- }
-
- //////////////// update
-
- private static final int UpdateAlways = 0;
- private static final int UpdateIfAbsent = 1;
- private static final int UpdateIfPresent = 2;
- private static final int UpdateIfEq = 3;
-
- private static boolean shouldUpdate(final int func, final Object prev, final Object expected) {
- switch (func) {
- case UpdateAlways: return true;
- case UpdateIfAbsent: return prev == null;
- case UpdateIfPresent: return prev != null;
- default: { // UpdateIfEq
- assert(expected != null);
- if (prev == null) {
- return false;
- }
- if (AllowNullValues && (prev == SpecialNull || expected == SpecialNull)) {
- return prev == SpecialNull && expected == SpecialNull;
- }
- return prev.equals(expected);
- }
- }
- }
-
- private static Object noUpdateResult(final int func, final Object prev) {
- return func == UpdateIfEq ? Boolean.FALSE : prev;
- }
-
- private static Object updateResult(final int func, final Object prev) {
- return func == UpdateIfEq ? Boolean.TRUE : prev;
- }
-
- private static int sizeDelta(final int func, final Object result, final Object newValue) {
- switch (func) {
- case UpdateAlways: {
- return (result != null ? -1 : 0) + (newValue != null ? 1 : 0);
- }
- case UpdateIfAbsent: {
- assert(newValue != null);
- return result != null ? 0 : 1;
- }
- case UpdateIfPresent: {
- return result == null ? 0 : (newValue != null ? 0 : -1);
- }
- default: { // UpdateIfEq
- return !((Boolean) result) ? 0 : (newValue != null ? 0 : -1);
- }
- }
- }
-
- @Override
- public V put(final K key, final V value) {
- return decodeNull(update(key, UpdateAlways, null, encodeNull(value)));
- }
-
- @Override
- public V putIfAbsent(final K key, final V value) {
- return decodeNull(update(key, UpdateIfAbsent, null, encodeNull(value)));
- }
-
- @Override
- public V replace(final K key, final V value) {
- return decodeNull(update(key, UpdateIfPresent, null, encodeNull(value)));
- }
-
- @Override
- public boolean replace(final K key, final V oldValue, final V newValue) {
- return (Boolean) update(key, UpdateIfEq, encodeNull(oldValue), encodeNull(newValue));
- }
-
- @Override
- public V remove(final Object key) {
- return decodeNull(update(key, UpdateAlways, null, null));
- }
-
- @Override
- public boolean remove(final Object key, final Object value) {
- if (key == null) {
- throw new NullPointerException();
- }
- if (!AllowNullValues && value == null) {
- return false;
- }
- return (Boolean) update(key, UpdateIfEq, encodeNull(value), null);
- }
-
- // manages the epoch
- private Object update(final Object key,
- final int func,
- final Object expected,
- final Object newValue) {
- final Comparable<? super K> k = comparable(key);
- int sd = 0;
- final Epoch.Ticket ticket = holderRef.beginMutation();
- try {
- final Object result = updateUnderRoot(key, k, func, expected, newValue, holderRef.mutable());
- sd = sizeDelta(func, result, newValue);
- return result;
- } finally {
- ticket.leave(sd);
- }
- }
-
- // manages updates to the root holder
- @SuppressWarnings("unchecked")
- private Object updateUnderRoot(final Object key,
- final Comparable<? super K> k,
- final int func,
- final Object expected,
- final Object newValue,
- final RootHolder<K,V> holder) {
-
- while (true) {
- final Node<K,V> right = holder.unsharedRight();
- if (right == null) {
- // key is not present
- if (!shouldUpdate(func, null, expected)) {
- return noUpdateResult(func, null);
- }
- if (newValue == null || attemptInsertIntoEmpty((K)key, newValue, holder)) {
- // nothing needs to be done, or we were successful, prev value is Absent
- return updateResult(func, null);
- }
- // else RETRY
- } else {
- final long ovl = right.shrinkOVL;
- if (isShrinkingOrUnlinked(ovl)) {
- right.waitUntilShrinkCompleted(ovl);
- // RETRY
- } else if (right == holder.right) {
- // this is the protected .right
- final Object vo = attemptUpdate(key, k, func, expected, newValue, holder, right, ovl);
- if (vo != SpecialRetry) {
- return vo;
- }
- // else RETRY
- }
- }
- }
- }
-
- private boolean attemptInsertIntoEmpty(final K key,
- final Object vOpt,
- final RootHolder<K,V> holder) {
- synchronized (holder) {
- if (holder.right == null) {
- holder.right = new Node<K,V>(key, 1, vOpt, holder, 0L, null, null);
- holder.height = 2;
- return true;
- } else {
- return false;
- }
- }
- }
-
- /** If successful returns the non-null previous value, SpecialNull for a
- * null previous value, or null if not previously in the map.
- * The caller should retry if this method returns SpecialRetry.
- */
- @SuppressWarnings("unchecked")
- private Object attemptUpdate(final Object key,
- final Comparable<? super K> k,
- final int func,
- final Object expected,
- final Object newValue,
- final Node<K,V> parent,
- final Node<K,V> node,
- final long nodeOVL) {
- // As the search progresses there is an implicit min and max assumed for the
- // branch of the tree rooted at node. A left rotation of a node x results in
- // the range of keys in the right branch of x being reduced, so if we are at a
- // node and we wish to traverse to one of the branches we must make sure that
- // the node has not undergone a rotation since arriving from the parent.
- //
- // A rotation of node can't screw us up once we have traversed to node's
- // child, so we don't need to build a huge transaction, just a chain of
- // smaller read-only transactions.
-
- assert (nodeOVL != UnlinkedOVL);
-
- final int cmp = k.compareTo(node.key);
- if (cmp == 0) {
- return attemptNodeUpdate(func, expected, newValue, parent, node);
- }
-
- final char dirToC = cmp < 0 ? Left : Right;
-
- while (true) {
- final Node<K,V> child = node.unsharedChild(dirToC);
-
- if (node.shrinkOVL != nodeOVL) {
- return SpecialRetry;
- }
-
- if (child == null) {
- // key is not present
- if (newValue == null) {
- // Removal is requested. Read of node.child occurred
- // while parent.child was valid, so we were not affected
- // by any shrinks.
- return noUpdateResult(func, null);
- } else {
- // Update will be an insert.
- final boolean success;
- final Node<K,V> damaged;
- synchronized (node) {
- // Validate that we haven't been affected by past
- // rotations. We've got the lock on node, so no future
- // rotations can mess with us.
- if (node.shrinkOVL != nodeOVL) {
- return SpecialRetry;
- }
-
- if (node.child(dirToC) != null) {
- // Lost a race with a concurrent insert. No need
- // to back up to the parent, but we must RETRY in
- // the outer loop of this method.
- success = false;
- damaged = null;
- } else {
- // We're valid. Does the user still want to
- // perform the operation?
- if (!shouldUpdate(func, null, expected)) {
- return noUpdateResult(func, null);
- }
-
- // Create a new leaf
- node.setChild(dirToC, new Node<K,V>((K)key, 1, newValue, node, 0L, null, null));
- success = true;
-
- // attempt to fix node.height while we've still got
- // the lock
- damaged = fixHeight_nl(node);
- }
- }
- if (success) {
- fixHeightAndRebalance(damaged);
- return updateResult(func, null);
- }
- // else RETRY
- }
- } else {
- // non-null child
- final long childOVL = child.shrinkOVL;
- if (isShrinkingOrUnlinked(childOVL)) {
- child.waitUntilShrinkCompleted(childOVL);
- // RETRY
- } else if (child != node.child(dirToC)) {
- // this second read is important, because it is protected
- // by childOVL
- // RETRY
- } else {
- // validate the read that our caller took to get to node
- if (node.shrinkOVL != nodeOVL) {
- return SpecialRetry;
- }
-
- // At this point we know that the traversal our parent took
- // to get to node is still valid. The recursive
- // implementation will validate the traversal from node to
- // child, so just prior to the nodeOVL validation both
- // traversals were definitely okay. This means that we are
- // no longer vulnerable to node shrinks, and we don't need
- // to validate nodeOVL any more.
- final Object vo = attemptUpdate(key, k, func, expected, newValue, node, child, childOVL);
- if (vo != SpecialRetry) {
- return vo;
- }
- // else RETRY
- }
- }
- }
- }
-
- /** parent will only be used for unlink, update can proceed even if parent
- * is stale.
- */
- private Object attemptNodeUpdate(final int func,
- final Object expected,
- final Object newValue,
- final Node<K,V> parent,
- final Node<K,V> node) {
- if (newValue == null) {
- // removal
- if (node.vOpt == null) {
- // This node is already removed, nothing to do.
- return noUpdateResult(func, null);
- }
- }
-
- if (newValue == null && (node.left == null || node.right == null)) {
- // potential unlink, get ready by locking the parent
- final Object prev;
- final Node<K,V> damaged;
- synchronized (parent) {
- if (isUnlinked(parent.shrinkOVL) || node.parent != parent) {
- return SpecialRetry;
- }
-
- synchronized (node) {
- prev = node.vOpt;
- if (!shouldUpdate(func, prev, expected)) {
- return noUpdateResult(func, prev);
- }
- if (prev == null) {
- return updateResult(func, prev);
- }
- if (!attemptUnlink_nl(parent, node)) {
- return SpecialRetry;
- }
- }
- // try to fix the parent while we've still got the lock
- damaged = fixHeight_nl(parent);
- }
- fixHeightAndRebalance(damaged);
- return updateResult(func, prev);
- } else {
- // potential update (including remove-without-unlink)
- synchronized (node) {
- // regular version changes don't bother us
- if (isUnlinked(node.shrinkOVL)) {
- return SpecialRetry;
- }
-
- final Object prev = node.vOpt;
- if (!shouldUpdate(func, prev, expected)) {
- return noUpdateResult(func, prev);
- }
-
- // retry if we now detect that unlink is possible
- if (newValue == null && (node.left == null || node.right == null)) {
- return SpecialRetry;
- }
-
- // update in-place
- node.vOpt = newValue;
-
- afterNodeUpdate_nl(node, newValue);
-
- return updateResult(func, prev);
- }
- }
- }
-
- protected void afterNodeUpdate_nl(Node<K,V> node, Object val) {
- }
-
- /** Does not adjust the size or any heights. */
- private boolean attemptUnlink_nl(final Node<K,V> parent, final Node<K,V> node) {
- // assert (Thread.holdsLock(parent));
- // assert (Thread.holdsLock(node));
- assert (!isUnlinked(parent.shrinkOVL));
-
- final Node<K,V> parentL = parent.left;
- final Node<K,V> parentR = parent.right;
- if (parentL != node && parentR != node) {
- // node is no longer a child of parent
- return false;
- }
-
- assert (!isUnlinked(node.shrinkOVL));
- assert (parent == node.parent);
-
- final Node<K,V> left = node.unsharedLeft();
- final Node<K,V> right = node.unsharedRight();
- if (left != null && right != null) {
- // splicing is no longer possible
- return false;
- }
- final Node<K,V> splice = left != null ? left : right;
-
- if (parentL == node) {
- parent.left = splice;
- } else {
- parent.right = splice;
- }
- if (splice != null) {
- splice.parent = parent;
- }
-
- node.shrinkOVL = UnlinkedOVL;
- node.vOpt = null;
-
- return true;
- }
-
- //////////////// NavigableMap stuff
-
- @Override
- public Map.Entry<K,V> pollFirstEntry() {
- return pollExtremeEntry(Left);
- }
-
- @Override
- public Map.Entry<K,V> pollLastEntry() {
- return pollExtremeEntry(Right);
- }
-
- private Map.Entry<K,V> pollExtremeEntry(final char dir) {
- final Epoch.Ticket ticket = holderRef.beginMutation();
- int sizeDelta = 0;
- try {
- final Map.Entry<K,V> prev = pollExtremeEntryUnderRoot(dir, holderRef.mutable());
- if (prev != null) {
- sizeDelta = -1;
- }
- return prev;
- } finally {
- ticket.leave(sizeDelta);
- }
- }
-
- private Map.Entry<K,V> pollExtremeEntryUnderRoot(final char dir, final RootHolder<K,V> holder) {
- while (true) {
- final Node<K,V> right = holder.unsharedRight();
- if (right == null) {
- // tree is empty, nothing to remove
- return null;
- } else {
- final long ovl = right.shrinkOVL;
- if (isShrinkingOrUnlinked(ovl)) {
- right.waitUntilShrinkCompleted(ovl);
- // RETRY
- } else if (right == holder.right) {
- // this is the protected .right
- final Map.Entry<K,V> result = attemptRemoveExtreme(dir, holder, right, ovl);
- if (result != SpecialRetry) {
- return result;
- }
- // else RETRY
- }
- }
- }
- }
-
- private Map.Entry<K,V> attemptRemoveExtreme(final char dir,
- final Node<K,V> parent,
- final Node<K,V> node,
- final long nodeOVL) {
- assert (nodeOVL != UnlinkedOVL);
-
- while (true) {
- final Node<K,V> child = node.unsharedChild(dir);
-
- if (nodeOVL != node.shrinkOVL) {
- return null;
- }
-
- if (child == null) {
- // potential unlink, get ready by locking the parent
- final Object vo;
- final Node<K,V> damaged;
- synchronized (parent) {
- if (isUnlinked(parent.shrinkOVL) || node.parent != parent) {
- return null;
- }
-
- synchronized (node) {
- vo = node.vOpt;
- if (node.child(dir) != null || !attemptUnlink_nl(parent, node)) {
- return null;
- }
- // success!
- }
- // try to fix parent.height while we've still got the lock
- damaged = fixHeight_nl(parent);
- }
- fixHeightAndRebalance(damaged);
- return new SimpleImmutableEntry<K,V>(node.key, decodeNull(vo));
- } else {
- // keep going down
- final long childOVL = child.shrinkOVL;
- if (isShrinkingOrUnlinked(childOVL)) {
- child.waitUntilShrinkCompleted(childOVL);
- // RETRY
- } else if (child != node.child(dir)) {
- // this second read is important, because it is protected
- // by childOVL
- // RETRY
- } else {
- // validate the read that our caller took to get to node
- if (node.shrinkOVL != nodeOVL) {
- return null;
- }
-
- final Map.Entry<K,V> result = attemptRemoveExtreme(dir, node, child, childOVL);
- if (result != null) {
- return result;
- }
- // else RETRY
- }
- }
- }
- }
-
-
-
- //////////////// tree balance and height info repair
-
- private static final int UnlinkRequired = -1;
- private static final int RebalanceRequired = -2;
- private static final int NothingRequired = -3;
-
- private int nodeCondition(final Node<K,V> node) {
- // Begin atomic.
-
- final Node<K,V> nL = node.left;
- final Node<K,V> nR = node.right;
-
- if ((nL == null || nR == null) && node.vOpt == null) {
- return UnlinkRequired;
- }
-
- final int hN = node.height;
- final int hL0 = height(nL);
- final int hR0 = height(nR);
-
- // End atomic. Since any thread that changes a node promises to fix
- // it, either our read was consistent (and a NothingRequired conclusion
- // is correct) or someone else has taken responsibility for either node
- // or one of its children.
-
- final int hNRepl = 1 + Math.max(hL0, hR0);
- final int bal = hL0 - hR0;
-
- if (bal < -1 || bal > 1) {
- return RebalanceRequired;
- }
-
- return hN != hNRepl ? hNRepl : NothingRequired;
- }
-
- private void fixHeightAndRebalance(Node<K,V> node) {
- while (node != null && node.parent != null) {
- final int condition = nodeCondition(node);
- if (condition == NothingRequired || isUnlinked(node.shrinkOVL)) {
- // nothing to do, or no point in fixing this node
- return;
- }
-
- if (condition != UnlinkRequired && condition != RebalanceRequired) {
- synchronized (node) {
- node = fixHeight_nl(node);
- }
- } else {
- final Node<K,V> nParent = node.parent;
- synchronized (nParent) {
- if (!isUnlinked(nParent.shrinkOVL) && node.parent == nParent) {
- synchronized (node) {
- node = rebalance_nl(nParent, node);
- }
- }
- // else RETRY
- }
- }
- }
- }
-
- /** Attempts to fix the height of a (locked) damaged node, returning the
- * lowest damaged node for which this thread is responsible. Returns null
- * if no more repairs are needed.
- */
- private Node<K,V> fixHeight_nl(final Node<K,V> node) {
- final int c = nodeCondition(node);
- switch (c) {
- case RebalanceRequired:
- case UnlinkRequired:
- // can't repair
- return node;
- case NothingRequired:
- // Any future damage to this node is not our responsibility.
- return null;
- default:
- node.height = c;
- // we've damaged our parent, but we can't fix it now
- return node.parent;
- }
- }
-
- /** nParent and n must be locked on entry. Returns a damaged node, or null
- * if no more rebalancing is necessary.
- */
- private Node<K,V> rebalance_nl(final Node<K,V> nParent, final Node<K,V> n) {
-
- final Node<K,V> nL = n.unsharedLeft();
- final Node<K,V> nR = n.unsharedRight();
-
- if ((nL == null || nR == null) && n.vOpt == null) {
- if (attemptUnlink_nl(nParent, n)) {
- // attempt to fix nParent.height while we've still got the lock
- return fixHeight_nl(nParent);
- } else {
- // retry needed for n
- return n;
- }
- }
-
- final int hN = n.height;
- final int hL0 = height(nL);
- final int hR0 = height(nR);
- final int hNRepl = 1 + Math.max(hL0, hR0);
- final int bal = hL0 - hR0;
-
- if (bal > 1) {
- return rebalanceToRight_nl(nParent, n, nL, hR0);
- } else if (bal < -1) {
- return rebalanceToLeft_nl(nParent, n, nR, hL0);
- } else if (hNRepl != hN) {
- // we've got more than enough locks to do a height change, no need to
- // trigger a retry
- n.height = hNRepl;
-
- // nParent is already locked, let's try to fix it too
- return fixHeight_nl(nParent);
- } else {
- // nothing to do
- return null;
- }
- }
-
- private Node<K,V> rebalanceToRight_nl(final Node<K,V> nParent,
- final Node<K,V> n,
- final Node<K,V> nL,
- final int hR0) {
- // L is too large, we will rotate-right. If L.R is taller
- // than L.L, then we will first rotate-left L.
- synchronized (nL) {
- final int hL = nL.height;
- if (hL - hR0 <= 1) {
- return n; // retry
- } else {
- final Node<K,V> nLR = nL.unsharedRight();
- final int hLL0 = height(nL.left);
- final int hLR0 = height(nLR);
- if (hLL0 >= hLR0) {
- // rotate right based on our snapshot of hLR
- return rotateRight_nl(nParent, n, nL, hR0, hLL0, nLR, hLR0);
- } else {
- synchronized (nLR) {
- // If our hLR snapshot is incorrect then we might
- // actually need to do a single rotate-right on n.
- final int hLR = nLR.height;
- if (hLL0 >= hLR) {
- return rotateRight_nl(nParent, n, nL, hR0, hLL0, nLR, hLR);
- } else {
- // If the underlying left balance would not be
- // sufficient to actually fix n.left, then instead
- // of rolling it into a double rotation we do it on
- // it's own. This may let us avoid rotating n at
- // all, but more importantly it avoids the creation
- // of damaged nodes that don't have a direct
- // ancestry relationship. The recursive call to
- // rebalanceToRight_nl in this case occurs after we
- // release the lock on nLR.
- //
- // We also need to avoid damaging n.left if post-
- // rotation it would be an unnecessary routing node.
- // Note that although our height snapshots might be
- // stale, their zero/non-zero state can't be.
- final int hLRL = height(nLR.left);
- final int b = hLL0 - hLRL;
- if (b >= -1 && b <= 1 && !((hLL0 == 0 || hLRL == 0) && nL.vOpt == null)) {
- // nParent.child.left won't be damaged after a double rotation
- return rotateRightOverLeft_nl(nParent, n, nL, hR0, hLL0, nLR, hLRL);
- }
- }
- }
- // focus on nL, if necessary n will be balanced later
- return rebalanceToLeft_nl(n, nL, nLR, hLL0);
- }
- }
- }
- }
-
- private Node<K,V> rebalanceToLeft_nl(final Node<K,V> nParent,
- final Node<K,V> n,
- final Node<K,V> nR,
- final int hL0) {
- synchronized (nR) {
- final int hR = nR.height;
- if (hL0 - hR >= -1) {
- return n; // retry
- } else {
- final Node<K,V> nRL = nR.unsharedLeft();
- final int hRL0 = height(nRL);
- final int hRR0 = height(nR.right);
- if (hRR0 >= hRL0) {
- return rotateLeft_nl(nParent, n, hL0, nR, nRL, hRL0, hRR0);
- } else {
- synchronized (nRL) {
- final int hRL = nRL.height;
- if (hRR0 >= hRL) {
- return rotateLeft_nl(nParent, n, hL0, nR, nRL, hRL, hRR0);
- } else {
- final int hRLR = height(nRL.right);
- final int b = hRR0 - hRLR;
- if (b >= -1 && b <= 1 && !((hRR0 == 0 || hRLR == 0) && nR.vOpt == null)) {
- return rotateLeftOverRight_nl(nParent, n, hL0, nR, nRL, hRR0, hRLR);
- }
- }
- }
- return rebalanceToRight_nl(n, nR, nRL, hRR0);
- }
- }
- }
- }
-
- private Node<K,V> rotateRight_nl(final Node<K,V> nParent,
- final Node<K,V> n,
- final Node<K,V> nL,
- final int hR,
- final int hLL,
- final Node<K,V> nLR,
- final int hLR) {
- final long nodeOVL = n.shrinkOVL;
-
- final Node<K,V> nPL = nParent.left;
-
- n.shrinkOVL = beginChange(nodeOVL);
-
- n.left = nLR;
- if (nLR != null) {
- nLR.parent = n;
- }
-
- nL.right = n;
- n.parent = nL;
-
- if (nPL == n) {
- nParent.left = nL;
- } else {
- nParent.right = nL;
- }
- nL.parent = nParent;
-
- // fix up heights links
- final int hNRepl = 1 + Math.max(hLR, hR);
- n.height = hNRepl;
- nL.height = 1 + Math.max(hLL, hNRepl);
-
- n.shrinkOVL = endChange(nodeOVL);
-
- // We have damaged nParent, n (now parent.child.right), and nL (now
- // parent.child). n is the deepest. Perform as many fixes as we can
- // with the locks we've got.
-
- // We've already fixed the height for n, but it might still be outside
- // our allowable balance range. In that case a simple fixHeight_nl
- // won't help.
- final int balN = hLR - hR;
- if (balN < -1 || balN > 1) {
- // we need another rotation at n
- return n;
- }
-
- // we've fixed balance and height damage for n, now handle
- // extra-routing node damage
- if ((nLR == null || hR == 0) && n.vOpt == null) {
- // we need to remove n and then repair
- return n;
- }
-
- // we've already fixed the height at nL, do we need a rotation here?
- final int balL = hLL - hNRepl;
- if (balL < -1 || balL > 1) {
- return nL;
- }
-
- // nL might also have routing node damage (if nL.left was null)
- if (hLL == 0 && nL.vOpt == null) {
- return nL;
- }
-
- // try to fix the parent height while we've still got the lock
- return fixHeight_nl(nParent);
- }
-
- private Node<K,V> rotateLeft_nl(final Node<K,V> nParent,
- final Node<K,V> n,
- final int hL,
- final Node<K,V> nR,
- final Node<K,V> nRL,
- final int hRL,
- final int hRR) {
- final long nodeOVL = n.shrinkOVL;
-
- final Node<K,V> nPL = nParent.left;
-
- n.shrinkOVL = beginChange(nodeOVL);
-
- // fix up n links, careful to be compatible with concurrent traversal for all but n
- n.right = nRL;
- if (nRL != null) {
- nRL.parent = n;
- }
-
- nR.left = n;
- n.parent = nR;
-
- if (nPL == n) {
- nParent.left = nR;
- } else {
- nParent.right = nR;
- }
- nR.parent = nParent;
-
- // fix up heights
- final int hNRepl = 1 + Math.max(hL, hRL);
- n.height = hNRepl;
- nR.height = 1 + Math.max(hNRepl, hRR);
-
- n.shrinkOVL = endChange(nodeOVL);
-
- final int balN = hRL - hL;
- if (balN < -1 || balN > 1) {
- return n;
- }
-
- if ((nRL == null || hL == 0) && n.vOpt == null) {
- return n;
- }
-
- final int balR = hRR - hNRepl;
- if (balR < -1 || balR > 1) {
- return nR;
- }
-
- if (hRR == 0 && nR.vOpt == null) {
- return nR;
- }
-
- return fixHeight_nl(nParent);
- }
-
- private Node<K,V> rotateRightOverLeft_nl(final Node<K,V> nParent,
- final Node<K,V> n,
- final Node<K,V> nL,
- final int hR,
- final int hLL,
- final Node<K,V> nLR,
- final int hLRL) {
- final long nodeOVL = n.shrinkOVL;
- final long leftOVL = nL.shrinkOVL;
-
- final Node<K,V> nPL = nParent.left;
- final Node<K,V> nLRL = nLR.unsharedLeft();
- final Node<K,V> nLRR = nLR.unsharedRight();
- final int hLRR = height(nLRR);
-
- n.shrinkOVL = beginChange(nodeOVL);
- nL.shrinkOVL = beginChange(leftOVL);
-
- // fix up n links, careful about the order!
- n.left = nLRR;
- if (nLRR != null) {
- nLRR.parent = n;
- }
-
- nL.right = nLRL;
- if (nLRL != null) {
- nLRL.parent = nL;
- }
-
- nLR.left = nL;
- nL.parent = nLR;
- nLR.right = n;
- n.parent = nLR;
-
- if (nPL == n) {
- nParent.left = nLR;
- } else {
- nParent.right = nLR;
- }
- nLR.parent = nParent;
-
- // fix up heights
- final int hNRepl = 1 + Math.max(hLRR, hR);
- n.height = hNRepl;
- final int hLRepl = 1 + Math.max(hLL, hLRL);
- nL.height = hLRepl;
- nLR.height = 1 + Math.max(hLRepl, hNRepl);
-
- n.shrinkOVL = endChange(nodeOVL);
- nL.shrinkOVL = endChange(leftOVL);
-
- // caller should have performed only a single rotation if nL was going
- // to end up damaged
- assert(Math.abs(hLL - hLRL) <= 1);
- assert(!((hLL == 0 || nLRL == null) && nL.vOpt == null));
-
- // We have damaged nParent, nLR (now parent.child), and n (now
- // parent.child.right). n is the deepest. Perform as many fixes as we
- // can with the locks we've got.
-
- // We've already fixed the height for n, but it might still be outside
- // our allowable balance range. In that case a simple fixHeight_nl
- // won't help.
- final int balN = hLRR - hR;
- if (balN < -1 || balN > 1) {
- // we need another rotation at n
- return n;
- }
-
- // n might also be damaged by being an unnecessary routing node
- if ((nLRR == null || hR == 0) && n.vOpt == null) {
- // repair involves splicing out n and maybe more rotations
- return n;
- }
-
- // we've already fixed the height at nLR, do we need a rotation here?
- final int balLR = hLRepl - hNRepl;
- if (balLR < -1 || balLR > 1) {
- return nLR;
- }
-
- // try to fix the parent height while we've still got the lock
- return fixHeight_nl(nParent);
- }
-
- private Node<K,V> rotateLeftOverRight_nl(final Node<K,V> nParent,
- final Node<K,V> n,
- final int hL,
- final Node<K,V> nR,
- final Node<K,V> nRL,
- final int hRR,
- final int hRLR) {
- final long nodeOVL = n.shrinkOVL;
- final long rightOVL = nR.shrinkOVL;
-
- final Node<K,V> nPL = nParent.left;
- final Node<K,V> nRLL = nRL.unsharedLeft();
- final Node<K,V> nRLR = nRL.unsharedRight();
- final int hRLL = height(nRLL);
-
- n.shrinkOVL = beginChange(nodeOVL);
- nR.shrinkOVL = beginChange(rightOVL);
-
- // fix up n links, careful about the order!
- n.right = nRLL;
- if (nRLL != null) {
- nRLL.parent = n;
- }
-
- nR.left = nRLR;
- if (nRLR != null) {
- nRLR.parent = nR;
- }
-
- nRL.right = nR;
- nR.parent = nRL;
- nRL.left = n;
- n.parent = nRL;
-
- if (nPL == n) {
- nParent.left = nRL;
- } else {
- nParent.right = nRL;
- }
- nRL.parent = nParent;
-
- // fix up heights
- final int hNRepl = 1 + Math.max(hL, hRLL);
- n.height = hNRepl;
- final int hRRepl = 1 + Math.max(hRLR, hRR);
- nR.height = hRRepl;
- nRL.height = 1 + Math.max(hNRepl, hRRepl);
-
- n.shrinkOVL = endChange(nodeOVL);
- nR.shrinkOVL = endChange(rightOVL);
-
- assert(Math.abs(hRR - hRLR) <= 1);
-
- final int balN = hRLL - hL;
- if (balN < -1 || balN > 1) {
- return n;
- }
- if ((nRLL == null || hL == 0) && n.vOpt == null) {
- return n;
- }
- final int balRL = hRRepl - hNRepl;
- if (balRL < -1 || balRL > 1) {
- return nRL;
- }
- return fixHeight_nl(nParent);
- }
-
- //////////////// Map views
-
- @Override
- public NavigableSet<K> keySet() {
- return navigableKeySet();
- }
-
- @Override
- public Set<Map.Entry<K,V>> entrySet() {
- return new EntrySet();
- }
-
- private class EntrySet extends AbstractSet<Map.Entry<K,V>> {
-
- @Override
- public int size() {
- return SnapTreeMap.this.size();
- }
-
- @Override
- public boolean isEmpty() {
- return SnapTreeMap.this.isEmpty();
- }
-
- @Override
- public void clear() {
- SnapTreeMap.this.clear();
- }
-
- @Override
- public boolean contains(final Object o) {
- if (!(o instanceof Map.Entry<?,?>)) {
- return false;
- }
- final Object k = ((Map.Entry<?,?>)o).getKey();
- final Object v = ((Map.Entry<?,?>)o).getValue();
- final Object actualVo = SnapTreeMap.this.getImpl(k);
- if (actualVo == null) {
- // no associated value
- return false;
- }
- final V actual = decodeNull(actualVo);
- return v == null ? actual == null : v.equals(actual);
- }
-
- @Override
- public boolean add(final Entry<K,V> e) {
- final Object v = encodeNull(e.getValue());
- return update(e.getKey(), UpdateAlways, null, v) != v;
- }
-
- @Override
- public boolean remove(final Object o) {
- if (!(o instanceof Map.Entry<?,?>)) {
- return false;
- }
- final Object k = ((Map.Entry<?,?>)o).getKey();
- final Object v = ((Map.Entry<?,?>)o).getValue();
- return SnapTreeMap.this.remove(k, v);
- }
-
- @Override
- public Iterator<Entry<K,V>> iterator() {
- return new EntryIter<K,V>(SnapTreeMap.this);
- }
- }
-
- private static class EntryIter<K,V> extends AbstractIter<K,V> implements Iterator<Map.Entry<K,V>> {
- private EntryIter(final SnapTreeMap<K,V> m) {
- super(m);
- }
-
- private EntryIter(final SnapTreeMap<K,V> m,
- final Comparable<? super K> minCmp,
- final boolean minIncl,
- final Comparable<? super K> maxCmp,
- final boolean maxIncl,
- final boolean descending) {
- super(m, minCmp, minIncl, maxCmp, maxIncl, descending);
- }
-
- @Override
- public Entry<K,V> next() {
- return nextNode();
- }
- }
-
- private static class KeyIter<K,V> extends AbstractIter<K,V> implements Iterator<K> {
- private KeyIter(final SnapTreeMap<K,V> m) {
- super(m);
- }
-
- private KeyIter(final SnapTreeMap<K,V> m,
- final Comparable<? super K> minCmp,
- final boolean minIncl,
- final Comparable<? super K> maxCmp,
- final boolean maxIncl,
- final boolean descending) {
- super(m, minCmp, minIncl, maxCmp, maxIncl, descending);
- }
-
- @Override
- public K next() {
- return nextNode().key;
- }
- }
-
- private static class AbstractIter<K,V> {
- private final SnapTreeMap<K,V> m;
- private final boolean descending;
- private final char forward;
- private final char reverse;
- private Node<K,V>[] path;
- private int depth = 0;
- private Node<K,V> mostRecentNode;
- private final K endKey;
-
- @SuppressWarnings("unchecked")
- AbstractIter(final SnapTreeMap<K,V> m) {
- this.m = m;
- this.descending = false;
- this.forward = Right;
- this.reverse = Left;
- final Node<K,V> root = m.holderRef.frozen().right;
- this.path = (Node<K,V>[]) new Node[1 + height(root)];
- this.endKey = null;
- pushFirst(root);
- }
-
- @SuppressWarnings("unchecked")
- AbstractIter(final SnapTreeMap<K,V> m,
- final Comparable<? super K> minCmp,
- final boolean minIncl,
- final Comparable<? super K> maxCmp,
- final boolean maxIncl,
- final boolean descending) {
- this.m = m;
- this.descending = descending;
- this.forward = !descending ? Right : Left;
- this.reverse = !descending ? Left : Right;
- final Comparable<? super K> fromCmp;
- final boolean fromIncl = !descending ? minIncl : maxIncl;
- final Comparable<? super K> toCmp;
- final boolean toIncl = !descending ? maxIncl : minIncl;
- if (!descending) {
- fromCmp = minCmp;
- toCmp = maxCmp;
- } else {
- fromCmp = maxCmp;
- toCmp = minCmp;
- }
-
- final Node<K,V> root = m.holderRef.frozen().right;
-
- if (toCmp != null) {
- this.endKey = (K) m.boundedExtreme(minCmp, minIncl, maxCmp, maxIncl, true, forward);
- if (this.endKey == null) {
- // no node satisfies the bound, nothing to iterate
- // ---------> EARLY EXIT
- return;
- }
- } else {
- this.endKey = null;
- }
-
- this.path = (Node<K,V>[]) new Node[1 + height(root)];
-
- if (fromCmp == null) {
- pushFirst(root);
- }
- else {
- pushFirst(root, fromCmp, fromIncl);
- if (depth > 0 && top().vOpt == null) {
- advance();
- }
- }
- }
-
- private int cmp(final Comparable<? super K> comparable, final K key) {
- final int c = comparable.compareTo(key);
- if (!descending) {
- return c;
- } else {
- return c == Integer.MIN_VALUE ? 1 : -c;
- }
- }
-
- private void pushFirst(Node<K,V> node) {
- while (node != null) {
- path(node);
- node = node.child(reverse);
- }
- }
-
- private void path(Node<K,V> node) {
- if (depth == path.length)
- path = Arrays.copyOf(path, depth + 2);
-
- path[depth++] = node;
- }
-
- private void pushFirst(Node<K,V> node, final Comparable<? super K> fromCmp, final boolean fromIncl) {
- while (node != null) {
- final int c = cmp(fromCmp, node.key);
- if (c > 0 || (c == 0 && !fromIncl)) {
- // everything we're interested in is on the right
- node = node.child(forward);
- }
- else {
- path(node);
- if (c == 0) {
- // start the iteration here
- return;
- }
- else {
- node = node.child(reverse);
- }
- }
- }
- }
-
- private Node<K,V> top() {
- return path[depth - 1];
- }
-
- private void advance() {
- do {
- final Node<K,V> t = top();
- if (endKey != null && endKey == t.key) {
- depth = 0;
- path = null;
- return;
- }
-
- final Node<K,V> fwd = t.child(forward);
- if (fwd != null) {
- pushFirst(fwd);
- } else {
- // keep going up until we pop a node that is a left child
- Node<K,V> popped;
- do {
- popped = path[--depth];
- } while (depth > 0 && popped == top().child(forward));
- }
-
- if (depth == 0) {
- // clear out the path so we don't pin too much stuff
- path = null;
- return;
- }
-
- // skip removed-but-not-unlinked entries
- } while (top().vOpt == null);
- }
-
- public boolean hasNext() {
- return depth > 0;
- }
-
- Node<K,V> nextNode() {
- if (depth == 0) {
- throw new NoSuchElementException();
- }
- mostRecentNode = top();
- advance();
- return mostRecentNode;
- }
-
- public void remove() {
- if (mostRecentNode == null) {
- throw new IllegalStateException();
- }
- m.remove(mostRecentNode.key);
- mostRecentNode = null;
- }
- }
-
- //////////////// navigable keySet
-
- @Override
- public NavigableSet<K> navigableKeySet() {
- return new KeySet<K>(this) {
- public Iterator<K> iterator() {
- return new KeyIter<K,V>(SnapTreeMap.this);
- }
- };
- }
-
- @Override
- public NavigableSet<K> descendingKeySet() {
- return descendingMap().navigableKeySet();
- }
-
- private abstract static class KeySet<K> extends AbstractSet<K> implements NavigableSet<K> {
-
- private final ConcurrentNavigableMap<K,?> map;
-
- protected KeySet(final ConcurrentNavigableMap<K,?> map) {
- this.map = map;
- }
-
- //////// basic Set stuff
-
- @Override
- abstract public Iterator<K> iterator();
-
- @Override
- public boolean contains(final Object o) { return map.containsKey(o); }
- @Override
- public boolean isEmpty() { return map.isEmpty(); }
- @Override
- public int size() { return map.size(); }
- @Override
- public boolean remove(final Object o) { return map.remove(o) != null; }
-
- //////// SortedSet stuff
-
- @Override
- public Comparator<? super K> comparator() { return map.comparator(); }
- @Override
- public K first() { return map.firstKey(); }
- @Override
- public K last() { return map.lastKey(); }
-
- //////// NavigableSet stuff
-
- @Override
- public K lower(final K k) { return map.lowerKey(k); }
- @Override
- public K floor(final K k) { return map.floorKey(k); }
- @Override
- public K ceiling(final K k) { return map.ceilingKey(k); }
- @Override
- public K higher(final K k) { return map.higherKey(k); }
-
- @Override
- public K pollFirst() { return map.pollFirstEntry().getKey(); }
- @Override
- public K pollLast() { return map.pollLastEntry().getKey(); }
-
- @Override
- public NavigableSet<K> descendingSet() { return map.descendingKeySet(); }
- @Override
- public Iterator<K> descendingIterator() { return map.descendingKeySet().iterator(); }
-
- @Override
- public NavigableSet<K> subSet(final K fromElement, final boolean minInclusive, final K toElement, final boolean maxInclusive) {
- return map.subMap(fromElement, minInclusive, toElement, maxInclusive).keySet();
- }
- @Override
- public NavigableSet<K> headSet(final K toElement, final boolean inclusive) {
- return map.headMap(toElement, inclusive).keySet();
- }
- @Override
- public NavigableSet<K> tailSet(final K fromElement, final boolean inclusive) {
- return map.tailMap(fromElement, inclusive).keySet();
- }
- @Override
- public SortedSet<K> subSet(final K fromElement, final K toElement) {
- return map.subMap(fromElement, toElement).keySet();
- }
- @Override
- public SortedSet<K> headSet(final K toElement) {
- return map.headMap(toElement).keySet();
- }
- @Override
- public SortedSet<K> tailSet(final K fromElement) {
- return map.tailMap(fromElement).keySet();
- }
- }
-
- //////////////// NavigableMap views
-
- @Override
- public ConcurrentNavigableMap<K,V> subMap(final K fromKey,
- final boolean fromInclusive,
- final K toKey,
- final boolean toInclusive) {
- final Comparable<? super K> fromCmp = comparable(fromKey);
- if (fromCmp.compareTo(toKey) > 0) {
- throw new IllegalArgumentException();
- }
- return new SubMap<K,V>(this, fromKey, fromCmp, fromInclusive, toKey, comparable(toKey), toInclusive, false);
- }
-
- @Override
- public ConcurrentNavigableMap<K,V> headMap(final K toKey, final boolean inclusive) {
- return new SubMap<K,V>(this, null, null, false, toKey, comparable(toKey), inclusive, false);
- }
-
- @Override
- public ConcurrentNavigableMap<K,V> tailMap(final K fromKey, final boolean inclusive) {
- return new SubMap<K,V>(this, fromKey, comparable(fromKey), inclusive, null, null, false, false);
- }
-
- @Override
- public ConcurrentNavigableMap<K,V> subMap(final K fromKey, final K toKey) {
- return subMap(fromKey, true, toKey, false);
- }
-
- @Override
- public ConcurrentNavigableMap<K,V> headMap(final K toKey) {
- return headMap(toKey, false);
- }
-
- @Override
- public ConcurrentNavigableMap<K,V> tailMap(final K fromKey) {
- return tailMap(fromKey, true);
- }
-
- @Override
- public ConcurrentNavigableMap<K,V> descendingMap() {
- return new SubMap(this, null, null, false, null, null, false, true);
- }
-
- private static class SubMap<K, V> extends AbstractMap<K, V> implements ConcurrentNavigableMap<K, V>, Serializable {
- /** */
- private static final long serialVersionUID = 0L;
-
- private final SnapTreeMap<K,V> m;
- private final K minKey;
- private transient Comparable<? super K> minCmp;
- private final boolean minIncl;
- private final K maxKey;
- private transient Comparable<? super K> maxCmp;
- private final boolean maxIncl;
- private final boolean descending;
-
- private SubMap(final SnapTreeMap<K,V> m,
- final K minKey,
- final Comparable<? super K> minCmp,
- final boolean minIncl,
- final K maxKey,
- final Comparable<? super K> maxCmp,
- final boolean maxIncl,
- final boolean descending) {
- this.m = m;
- this.minKey = minKey;
- this.minCmp = minCmp;
- this.minIncl = minIncl;
- this.maxKey = maxKey;
- this.maxCmp = maxCmp;
- this.maxIncl = maxIncl;
- this.descending = descending;
- }
-
- // TODO: clone
-
- private boolean tooLow(final K key) {
- if (minCmp == null) {
- return false;
- } else {
- final int c = minCmp.compareTo(key);
- return c > 0 || (c == 0 && !minIncl);
- }
- }
-
- private boolean tooHigh(final K key) {
- if (maxCmp == null) {
- return false;
- } else {
- final int c = maxCmp.compareTo(key);
- return c < 0 || (c == 0 && !maxIncl);
- }
- }
-
- private boolean inRange(final K key) {
- return !tooLow(key) && !tooHigh(key);
- }
-
- private void requireInRange(final K key) {
- if (key == null) {
- throw new NullPointerException();
- }
- if (!inRange(key)) {
- throw new IllegalArgumentException();
- }
- }
-
- private char minDir() {
- return descending ? Right : Left;
- }
-
- private char maxDir() {
- return descending ? Left : Right;
- }
-
- //////// AbstractMap
-
- @Override
- public boolean isEmpty() {
- return m.boundedExtreme(minCmp, minIncl, maxCmp, maxIncl, true, Left) == null;
- }
-
- @Override
- public int size() {
- final Node<K,V> root = m.holderRef.frozen().right;
- return Node.computeFrozenSize(root, minCmp, minIncl, maxCmp, maxIncl);
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public boolean containsKey(final Object key) {
- if (key == null) {
- throw new NullPointerException();
- }
- final K k = (K) key;
- return inRange(k) && m.containsKey(k);
- }
-
- @Override
- public boolean containsValue(final Object value) {
- /
<TRUNCATED>
[42/50] [abbrv] ignite git commit: IGNITE-9084 Fixed error handling
for historical rebalance - Fixes #4437.
Posted by ag...@apache.org.
IGNITE-9084 Fixed error handling for historical rebalance - Fixes #4437.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a232b88f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a232b88f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a232b88f
Branch: refs/heads/ignite-5960
Commit: a232b88fe823a9823b86a0efd93c259e0cf0f0bc
Parents: d5432c0
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Thu Sep 6 18:03:50 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Sep 6 18:03:50 2018 +0300
----------------------------------------------------------------------
.../communication/GridIoMessageFactory.java | 7 +
.../processors/cache/GridCacheIoManager.java | 3 +-
.../dht/preloader/GridDhtPartitionDemander.java | 14 +-
.../dht/preloader/GridDhtPartitionSupplier.java | 69 +++++--
.../GridDhtPartitionSupplyMessage.java | 6 +-
.../GridDhtPartitionSupplyMessageV2.java | 153 +++++++++++++++
.../persistence/GridCacheOffheapManager.java | 10 +
.../db/wal/IgniteWalRebalanceTest.java | 185 ++++++++++++++++++-
8 files changed, 427 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a232b88f/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 8dddd8b..2970e71 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -54,6 +54,8 @@ import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
import org.apache.ignite.internal.processors.cache.WalStateAckMessage;
import org.apache.ignite.internal.processors.cache.binary.MetadataRequestMessage;
import org.apache.ignite.internal.processors.cache.binary.MetadataResponseMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.LatchAckMessage;
import org.apache.ignite.internal.processors.cache.distributed.GridCacheTtlUpdateRequest;
import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryRequest;
import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryResponse;
@@ -889,6 +891,11 @@ public class GridIoMessageFactory implements MessageFactory {
break;
+ case 120:
+ msg = new GridDhtPartitionSupplyMessageV2();
+
+ break;
+
case 124:
msg = new GridMessageCollection<>();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a232b88f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 0134421..1e25c93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -914,7 +914,8 @@ public class GridCacheIoManager extends GridCacheSharedManagerAdapter {
break;
- case 114: {
+ case 114:
+ case 120: {
processMessage(nodeId, msg, c);// Will be handled by Rebalance Demander.
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a232b88f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 851fcc9..3c1090f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -688,10 +688,20 @@ public class GridDhtPartitionDemander {
if (log.isDebugEnabled())
log.debug("Received supply message [grp=" + grp.cacheOrGroupName() + ", msg=" + supply + ']');
- // Check whether there were class loading errors on unmarshal
+ // Check whether there were error during supply message unmarshalling process.
if (supply.classError() != null) {
U.warn(log, "Rebalancing from node cancelled [grp=" + grp.cacheOrGroupName() + ", node=" + nodeId +
- "]. Class got undeployed during preloading: " + supply.classError());
+ "]. Supply message couldn't be unmarshalled: " + supply.classError());
+
+ fut.cancel(nodeId);
+
+ return;
+ }
+
+ // Check whether there were error during supplying process.
+ if (supply.error() != null) {
+ U.warn(log, "Rebalancing from node cancelled [grp=" + grp.cacheOrGroupName() + ", node=" + nodeId +
+ "]. Supplier has failed with error: " + supply.error());
fut.cancel(nodeId);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a232b88f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 524d02d..2090c07 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -218,9 +218,11 @@ class GridDhtPartitionSupplier {
if (node == null)
return;
- try {
- SupplyContext sctx;
+ IgniteRebalanceIterator iter = null;
+
+ SupplyContext sctx = null;
+ try {
synchronized (scMap) {
sctx = scMap.remove(contextId);
@@ -229,7 +231,7 @@ class GridDhtPartitionSupplier {
scMap.put(contextId, sctx);
if (log.isDebugEnabled())
- log.debug("Stale demand message [grp=" + grp.cacheOrGroupName()
+ log.debug("Stale demand message [cache=" + grp.cacheOrGroupName()
+ ", actualContext=" + sctx
+ ", from=" + nodeId
+ ", demandMsg=" + d + "]");
@@ -241,7 +243,7 @@ class GridDhtPartitionSupplier {
// Demand request should not contain empty partitions if no supply context is associated with it.
if (sctx == null && (d.partitions() == null || d.partitions().isEmpty())) {
if (log.isDebugEnabled())
- log.debug("Empty demand message [grp=" + grp.cacheOrGroupName()
+ log.debug("Empty demand message [cache=" + grp.cacheOrGroupName()
+ ", from=" + nodeId
+ ", topicId=" + topicId
+ ", demandMsg=" + d + "]");
@@ -272,8 +274,6 @@ class GridDhtPartitionSupplier {
d.topologyVersion(),
grp.deploymentEnabled());
- IgniteRebalanceIterator iter;
-
Set<Integer> remainingParts;
if (sctx == null || sctx.iterator == null) {
@@ -452,13 +452,56 @@ class GridDhtPartitionSupplier {
", topology=" + demTop + ", rebalanceId=" + d.rebalanceId() +
", topicId=" + topicId + "]");
}
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to send partition supply message to node: " + nodeId, e);
- }
- catch (IgniteSpiException e) {
- if (log.isDebugEnabled())
- log.debug("Failed to send message to node (current node is stopping?) [node=" + node.id() +
- ", msg=" + e.getMessage() + ']');
+ catch (Throwable t) {
+ if (grp.shared().kernalContext().isStopping())
+ return;
+
+ // Sending supply messages with error requires new protocol.
+ boolean sendErrMsg = node.version().compareTo(GridDhtPartitionSupplyMessageV2.AVAILABLE_SINCE) >= 0;
+
+ if (t instanceof IgniteSpiException) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to send message to node (current node is stopping?) [node=" + node.id() +
+ ", msg=" + t.getMessage() + ']');
+
+ sendErrMsg = false;
+ }
+ else
+ U.error(log, "Failed to continue supplying process for " +
+ "[cache=" + grp.cacheOrGroupName() + ", node=" + nodeId
+ + ", topicId=" + contextId.get2() + ", topVer=" + contextId.get3() + "]", t);
+
+ try {
+ if (sctx != null)
+ clearContext(sctx, log);
+ else if (iter != null)
+ iter.close();
+ }
+ catch (Throwable t1) {
+ U.error(log, "Failed to cleanup supplying context " +
+ "[cache=" + grp.cacheOrGroupName() + ", node=" + nodeId
+ + ", topicId=" + contextId.get2() + ", topVer=" + contextId.get3() + "]", t1);
+ }
+
+ if (!sendErrMsg)
+ return;
+
+ try {
+ GridDhtPartitionSupplyMessageV2 errMsg = new GridDhtPartitionSupplyMessageV2(
+ d.rebalanceId(),
+ grp.groupId(),
+ d.topologyVersion(),
+ grp.deploymentEnabled(),
+ t
+ );
+
+ reply(node, d, errMsg, contextId);
+ }
+ catch (Throwable t1) {
+ U.error(log, "Failed to send supply error message for " +
+ "[cache=" + grp.cacheOrGroupName() + ", node=" + nodeId
+ + ", topicId=" + contextId.get2() + ", topVer=" + contextId.get3() + "]", t1);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a232b88f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
index 4ecffc4..284700a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
@@ -89,10 +89,12 @@ public class GridDhtPartitionSupplyMessage extends GridCacheGroupIdMessage imple
* @param topVer Topology version.
* @param addDepInfo Deployment info flag.
*/
- GridDhtPartitionSupplyMessage(long rebalanceId,
+ GridDhtPartitionSupplyMessage(
+ long rebalanceId,
int grpId,
AffinityTopologyVersion topVer,
- boolean addDepInfo) {
+ boolean addDepInfo
+ ) {
this.grpId = grpId;
this.rebalanceId = rebalanceId;
this.topVer = topVer;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a232b88f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
new file mode 100644
index 0000000..a775766
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Supply message with supplier error transfer support.
+ */
+public class GridDhtPartitionSupplyMessageV2 extends GridDhtPartitionSupplyMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Available since. */
+ public static final IgniteProductVersion AVAILABLE_SINCE = IgniteProductVersion.fromString("2.7.0");
+
+ /** Supplying process error. */
+ @GridDirectTransient
+ private Throwable err;
+
+ /** Supplying process error bytes. */
+ private byte[] errBytes;
+
+ /**
+ * Default constructor.
+ */
+ public GridDhtPartitionSupplyMessageV2() {
+ }
+
+ /**
+ * @param rebalanceId Rebalance id.
+ * @param grpId Group id.
+ * @param topVer Topology version.
+ * @param addDepInfo Add dep info.
+ * @param err Supply process error.
+ */
+ public GridDhtPartitionSupplyMessageV2(
+ long rebalanceId,
+ int grpId,
+ AffinityTopologyVersion topVer,
+ boolean addDepInfo,
+ Throwable err
+ ) {
+ super(rebalanceId, grpId, topVer, addDepInfo);
+
+ this.err = err;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {
+ super.prepareMarshal(ctx);
+
+ if (err != null && errBytes == null)
+ errBytes = U.marshal(ctx, err);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr) throws IgniteCheckedException {
+ super.finishUnmarshal(ctx, ldr);
+
+ if (errBytes != null && err == null)
+ err = U.unmarshal(ctx, errBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!super.writeTo(buf, writer))
+ return false;
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 12:
+ if (!writer.writeByteArray("errBytes", errBytes))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ if (!super.readFrom(buf, reader))
+ return false;
+
+ switch (reader.state()) {
+ case 12:
+ errBytes = reader.readByteArray("errBytes");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(GridDhtPartitionSupplyMessageV2.class);
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Throwable error() {
+ return err;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return 120;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 13;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a232b88f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 4c45352..199efcb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -880,6 +880,10 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
/** Flag indicates that partition belongs to current {@link #next} is finished and no longer needs to rebalance. */
private boolean reachedPartitionEnd;
+ /** Flag indicates that update counters for requested partitions have been reached and done.
+ * It means that no further iteration is needed. */
+ private boolean doneAllPartitions;
+
/**
* @param grp Cache context.
* @param walIt WAL iterator.
@@ -953,6 +957,9 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
doneParts.add(next.partitionId());
reachedPartitionEnd = false;
+
+ if (doneParts.size() == partMap.size())
+ doneAllPartitions = true;
}
advance();
@@ -1011,6 +1018,9 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
private void advance() {
next = null;
+ if (doneAllPartitions)
+ return;
+
while (true) {
if (entryIt != null) {
while (entryIt.hasNext()) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a232b88f/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
index d4f6f0c..57565bf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
@@ -17,6 +17,10 @@
package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.OpenOption;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -38,18 +42,31 @@ import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCachePreloader;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteBiPredicate;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.spi.IgniteSpiException;
-import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Assert;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
/**
@@ -62,6 +79,9 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
/** Partitions count. */
private static final int PARTS_CNT = 32;
+ /** Block message predicate to set to Communication SPI in node configuration. */
+ private IgniteBiPredicate<ClusterNode, Message> blockMessagePredicate;
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
System.setProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0"); //to make all rebalance wal-based
@@ -92,6 +112,12 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
cfg.setCommunicationSpi(new WalRebalanceCheckingCommunicationSpi());
+ if (blockMessagePredicate != null) {
+ TestRecordingCommunicationSpi spi = (TestRecordingCommunicationSpi) cfg.getCommunicationSpi();
+
+ spi.blockMessages(blockMessagePredicate);
+ }
+
return cfg;
}
@@ -227,6 +253,8 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
cache.put(k, new IndexedObject(k - 1));
}
+ forceCheckpoint();
+
stopAllGrids();
IgniteEx ig0 = (IgniteEx) startGrids(2);
@@ -240,6 +268,8 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
for (int k = 0; k < entryCnt; k++)
cache.put(k, new IndexedObject(k));
+ forceCheckpoint();
+
// This node should rebalance data from other nodes and shouldn't have WAL history.
Ignite ignite = startGrid(2);
@@ -258,6 +288,8 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
cache.remove(k);
}
+ forceCheckpoint();
+
// Stop grids which have actual WAL history.
stopGrid(0);
@@ -309,6 +341,8 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
cache.put(k, new IndexedObject(k - 1));
}
+ forceCheckpoint();
+
stopAllGrids();
// Rewrite data with globally disabled WAL.
@@ -325,6 +359,8 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
for (int k = 0; k < entryCnt; k++)
cache.put(k, new IndexedObject(k));
+ forceCheckpoint();
+
crd.cluster().enableWal(CACHE_NAME);
// This node shouldn't rebalance data using WAL, because it was disabled on other nodes.
@@ -365,6 +401,100 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
}
/**
+ * Tests that cache rebalance is cancelled if supplyer node got exception during iteration over WAL.
+ *
+ * @throws Exception If failed.
+ */
+ public void testRebalanceCancelOnSupplyError() throws Exception {
+ // Prepare some data.
+ IgniteEx crd = (IgniteEx) startGrids(3);
+
+ crd.cluster().active(true);
+
+ final int entryCnt = PARTS_CNT * 10;
+
+ {
+ IgniteCache<Object, Object> cache = crd.cache(CACHE_NAME);
+
+ for (int k = 0; k < entryCnt; k++)
+ cache.put(k, new IndexedObject(k - 1));
+ }
+
+ forceCheckpoint();
+
+ stopAllGrids();
+
+ // Rewrite data to trigger further rebalance.
+ IgniteEx supplierNode = (IgniteEx) startGrid(0);
+
+ supplierNode.cluster().active(true);
+
+ IgniteCache<Object, Object> cache = supplierNode.cache(CACHE_NAME);
+
+ for (int k = 0; k < entryCnt; k++)
+ cache.put(k, new IndexedObject(k));
+
+ forceCheckpoint();
+
+ final int groupId = supplierNode.cachex(CACHE_NAME).context().groupId();
+
+ // Delay rebalance process for specified group.
+ blockMessagePredicate = (node, msg) -> {
+ if (msg instanceof GridDhtPartitionDemandMessage)
+ return ((GridDhtPartitionDemandMessage) msg).groupId() == groupId;
+
+ return false;
+ };
+
+ IgniteEx demanderNode = startGrid(2);
+
+ AffinityTopologyVersion curTopVer = demanderNode.context().discovery().topologyVersionEx();
+
+ // Wait for rebalance process start on demander node.
+ final GridCachePreloader preloader = demanderNode.cachex(CACHE_NAME).context().group().preloader();
+
+ GridTestUtils.waitForCondition(() ->
+ ((GridDhtPartitionDemander.RebalanceFuture) preloader.rebalanceFuture()).topologyVersion().equals(curTopVer),
+ getTestTimeout()
+ );
+
+ // Inject I/O factory which can throw exception during WAL read on supplier node.
+ FailingIOFactory ioFactory = new FailingIOFactory(new RandomAccessFileIOFactory());
+
+ ((FileWriteAheadLogManager) supplierNode.cachex(CACHE_NAME).context().shared().wal()).setFileIOFactory(ioFactory);
+
+ ioFactory.throwExceptionOnWalRead();
+
+ // Resume rebalance process.
+ TestRecordingCommunicationSpi spi = (TestRecordingCommunicationSpi) demanderNode.configuration().getCommunicationSpi();
+
+ spi.stopBlock();
+
+ // Wait till rebalance will be failed and cancelled.
+ Boolean result = preloader.rebalanceFuture().get();
+
+ Assert.assertEquals("Rebalance should be cancelled on demander node: " + preloader.rebalanceFuture(), false, result);
+
+ // Stop blocking messages and fail WAL during read.
+ blockMessagePredicate = null;
+
+ ioFactory.reset();
+
+ // Start last grid and wait for rebalance.
+ startGrid(1);
+
+ awaitPartitionMapExchange();
+
+ // Check data consistency.
+ for (Ignite ig : G.allGrids()) {
+ IgniteCache<Object, Object> cache1 = ig.cache(CACHE_NAME);
+
+ for (int k = 0; k < entryCnt; k++)
+ assertEquals(new IndexedObject(k), cache1.get(k));
+ }
+ }
+
+ /**
*
*/
private static class IndexedObject {
@@ -409,7 +539,7 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
/**
* Wrapper of communication spi to detect on what topology versions WAL rebalance has happened.
*/
- public static class WalRebalanceCheckingCommunicationSpi extends TcpCommunicationSpi {
+ public static class WalRebalanceCheckingCommunicationSpi extends TestRecordingCommunicationSpi {
/** (Group ID, Set of topology versions). */
private static final Map<Integer, Set<Long>> topVers = new HashMap<>();
@@ -464,4 +594,55 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
super.sendMessage(node, msg, ackC);
}
}
+
+ /**
+ *
+ */
+ static class FailingIOFactory implements FileIOFactory {
+ /** Fail read operations. */
+ private volatile boolean failRead;
+
+ /** Delegate. */
+ private final FileIOFactory delegate;
+
+ /**
+ * @param delegate Delegate.
+ */
+ FailingIOFactory(FileIOFactory delegate) {
+ this.delegate = delegate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file) throws IOException {
+ return create(file, CREATE, WRITE, READ);
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+ FileIO delegateIO = delegate.create(file, modes);
+
+ if (file.getName().endsWith(".wal") && failRead)
+ return new FileIODecorator(delegateIO) {
+ @Override public int read(ByteBuffer destBuf) throws IOException {
+ throw new IgniteException("Test exception.");
+ }
+ };
+
+ return delegateIO;
+ }
+
+ /**
+ *
+ */
+ public void throwExceptionOnWalRead() {
+ failRead = true;
+ }
+
+ /**
+ *
+ */
+ public void reset() {
+ failRead = false;
+ }
+ }
}
\ No newline at end of file
[21/50] [abbrv] ignite git commit: IGNITE-9448 Updated ZooKeeper
version to 3.4.13 - Fixes #4671.
Posted by ag...@apache.org.
IGNITE-9448 Updated ZooKeeper version to 3.4.13 - Fixes #4671.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/dfaffe2a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/dfaffe2a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/dfaffe2a
Branch: refs/heads/ignite-5960
Commit: dfaffe2af8f5e3b51eff3cfe426733311c2c100f
Parents: 625e261
Author: vd-pyatkov <vp...@gridgain.com>
Authored: Tue Sep 4 15:55:41 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Sep 4 15:55:41 2018 +0300
----------------------------------------------------------------------
.../tcp/ipfinder/zk/ZookeeperIpFinderTest.java | 3 +-
.../zk/curator/FixedTestingQuorumPeerMain.java | 87 +++++++
.../tcp/ipfinder/zk/curator/TestingCluster.java | 237 +++++++++++++++++++
.../zk/curator/TestingZooKeeperServer.java | 169 +++++++++++++
.../ZookeeperDiscoverySpiAbstractTestSuite.java | 2 +-
.../zk/ZookeeperDiscoverySpiTestSuite2.java | 2 +-
.../zk/internal/ZookeeperClientTest.java | 2 +-
.../zk/internal/ZookeeperDiscoverySpiTest.java | 4 +-
parent/pom.xml | 2 +-
9 files changed, 501 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/dfaffe2a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/ZookeeperIpFinderTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/ZookeeperIpFinderTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/ZookeeperIpFinderTest.java
index f85c6a3c..ec5630f 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/ZookeeperIpFinderTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/ZookeeperIpFinderTest.java
@@ -26,7 +26,6 @@ import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.curator.retry.RetryNTimes;
import org.apache.curator.test.InstanceSpec;
-import org.apache.curator.test.TestingCluster;
import org.apache.curator.utils.CloseableUtils;
import org.apache.ignite.Ignite;
import org.apache.ignite.configuration.IgniteConfiguration;
@@ -34,6 +33,7 @@ import org.apache.ignite.events.Event;
import org.apache.ignite.events.EventType;
import org.apache.ignite.lang.IgniteBiPredicate;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Rule;
@@ -85,6 +85,7 @@ public class ZookeeperIpFinderTest extends GridCommonAbstractTest {
// start the ZK cluster
zkCluster = new TestingCluster(ZK_CLUSTER_SIZE);
+
zkCluster.start();
// start the Curator client so we can perform assertions on the ZK state later
http://git-wip-us.apache.org/repos/asf/ignite/blob/dfaffe2a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/FixedTestingQuorumPeerMain.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/FixedTestingQuorumPeerMain.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/FixedTestingQuorumPeerMain.java
new file mode 100644
index 0000000..e0a416e
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/FixedTestingQuorumPeerMain.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator;
+
+import org.apache.curator.test.ZooKeeperMainFace;
+import org.apache.zookeeper.server.ServerCnxnFactory;
+import org.apache.zookeeper.server.quorum.QuorumPeer;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.channels.ServerSocketChannel;
+
+/**
+ */
+public class FixedTestingQuorumPeerMain extends QuorumPeerMain implements ZooKeeperMainFace {
+ @Override public void runFromConfig(QuorumPeerConfig config) throws IOException {
+ quorumPeer = QuorumPeer.testingQuorumPeer();
+ super.runFromConfig(config);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void kill() {
+ try {
+ if (quorumPeer != null) {
+ Field cnxnFactoryField = QuorumPeer.class.getDeclaredField("cnxnFactory");
+
+ cnxnFactoryField.setAccessible(true);
+
+ ServerCnxnFactory cnxnFactory = (ServerCnxnFactory)cnxnFactoryField.get(quorumPeer);
+
+ cnxnFactory.closeAll();
+
+ Field ssField = cnxnFactory.getClass().getDeclaredField("ss");
+
+ ssField.setAccessible(true);
+
+ ServerSocketChannel ss = (ServerSocketChannel)ssField.get(cnxnFactory);
+
+ ss.close();
+ }
+ close();
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public QuorumPeer getQuorumPeer() {
+ return quorumPeer;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IOException {
+ if (quorumPeer != null)
+ quorumPeer.shutdown();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void blockUntilStarted() throws Exception {
+ while (quorumPeer == null) {
+ try {
+ Thread.sleep(100);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/dfaffe2a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingCluster.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingCluster.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingCluster.java
new file mode 100644
index 0000000..c6e8d72
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingCluster.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.curator.test.ByteCodeRewrite;
+import org.apache.curator.test.InstanceSpec;
+import org.apache.curator.test.QuorumConfigBuilder;
+import org.apache.zookeeper.ZooKeeper;
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Manages an internally running ensemble of ZooKeeper servers. FOR TESTING PURPOSES ONLY.
+ * This class is a copy of {{org.apache.curator.test.TestingCluster}},
+ * but have very small change, that allow to run testing cluster with ZooKeeper 2.4.13 ver.
+ */
+public class TestingCluster implements Closeable {
+ static {
+ ByteCodeRewrite.apply();
+ }
+
+ /** Servers. */
+ private final List<TestingZooKeeperServer> servers;
+
+ /**
+ * Creates an ensemble comprised of <code>n</code> servers. Each server will use
+ * a temp directory and random ports
+ *
+ * @param instanceQty number of servers to create in the ensemble
+ */
+ public TestingCluster(int instanceQty) {
+ this(makeSpecs(instanceQty));
+ }
+
+ /**
+ * Creates an ensemble using the given server specs
+ *
+ * @param specs the server specs
+ */
+ public TestingCluster(InstanceSpec... specs) {
+ this(listToMap(ImmutableList.copyOf(specs)));
+ }
+
+ /**
+ * Creates an ensemble using the given server specs
+ *
+ * @param specs the server specs
+ */
+ public TestingCluster(Collection<InstanceSpec> specs) {
+ this(listToMap(specs));
+ }
+
+ /**
+ * Creates an ensemble using the given server specs
+ *
+ * @param specs map of an instance spec to its set of quorum instances. Allows simulation of an ensemble with
+ * instances having different config peers
+ */
+ public TestingCluster(Map<InstanceSpec, Collection<InstanceSpec>> specs) {
+ ImmutableList.Builder<TestingZooKeeperServer> serverBuilder = ImmutableList.builder();
+ for (Map.Entry<InstanceSpec, Collection<InstanceSpec>> entry : specs.entrySet()) {
+ List<InstanceSpec> instanceSpecs = Lists.newArrayList(entry.getValue());
+ int index = instanceSpecs.indexOf(entry.getKey());
+ Preconditions.checkState(index >= 0, entry.getKey() + " not found in specs");
+ QuorumConfigBuilder builder = new QuorumConfigBuilder(instanceSpecs);
+ serverBuilder.add(new TestingZooKeeperServer(builder, index));
+ }
+ servers = serverBuilder.build();
+ }
+
+ /**
+ * Returns the set of servers in the ensemble
+ *
+ * @return set of servers
+ */
+ public Collection<InstanceSpec> getInstances() {
+ Iterable<InstanceSpec> transformed = Iterables.transform
+ (
+ servers,
+ new Function<TestingZooKeeperServer, InstanceSpec>() {
+ @Override
+ public InstanceSpec apply(TestingZooKeeperServer server) {
+ return server.getInstanceSpec();
+ }
+ }
+ );
+ return Lists.newArrayList(transformed);
+ }
+
+ public List<TestingZooKeeperServer> getServers() {
+ return Lists.newArrayList(servers);
+ }
+
+ /**
+ * Returns the connection string to pass to the ZooKeeper constructor
+ *
+ * @return connection string
+ */
+ public String getConnectString() {
+ StringBuilder str = new StringBuilder();
+ for (InstanceSpec spec : getInstances()) {
+ if (str.length() > 0)
+ str.append(",");
+
+ str.append(spec.getConnectString());
+ }
+ return str.toString();
+ }
+
+ /**
+ * Start the ensemble. The cluster must be started before use.
+ *
+ * @throws Exception errors
+ */
+ public void start() throws Exception {
+ for (TestingZooKeeperServer server : servers)
+ server.start();
+
+ }
+
+ /**
+ * Shutdown the ensemble WITHOUT freeing resources, etc.
+ */
+ public void stop() throws IOException {
+ for (TestingZooKeeperServer server : servers)
+ server.stop();
+
+ }
+
+ /**
+ * Shutdown the ensemble, free resources, etc. If temp directories were used, they
+ * are deleted. You should call this in a <code>finally</code> block.
+ *
+ * @throws IOException errors
+ */
+ @Override public void close() throws IOException {
+ for (TestingZooKeeperServer server : servers)
+ server.close();
+ }
+
+ /**
+ * Kills the given server. This simulates the server unexpectedly crashing
+ *
+ * @param instance server to kill
+ * @return true if the instance was found
+ * @throws Exception errors
+ */
+ public boolean killServer(InstanceSpec instance) throws Exception {
+ for (TestingZooKeeperServer server : servers) {
+ if (server.getInstanceSpec().equals(instance)) {
+ server.kill();
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Restart the given server of the cluster
+ *
+ * @param instance server instance
+ * @return true of the server was found
+ * @throws Exception errors
+ */
+ public boolean restartServer(InstanceSpec instance) throws Exception {
+ for (TestingZooKeeperServer server : servers) {
+ if (server.getInstanceSpec().equals(instance)) {
+ server.restart();
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Given a ZooKeeper instance, returns which server it is connected to
+ *
+ * @param client ZK instance
+ * @return the server
+ * @throws Exception errors
+ */
+ public InstanceSpec findConnectionInstance(ZooKeeper client) throws Exception {
+ Method m = client.getClass().getDeclaredMethod("testableRemoteSocketAddress");
+ m.setAccessible(true);
+ InetSocketAddress address = (InetSocketAddress)m.invoke(client);
+ if (address != null) {
+ for (TestingZooKeeperServer server : servers) {
+ if (server.getInstanceSpec().getPort() == address.getPort())
+ return server.getInstanceSpec();
+ }
+ }
+
+ return null;
+ }
+
+ private static Map<InstanceSpec, Collection<InstanceSpec>> makeSpecs(int instanceQty) {
+ ImmutableList.Builder<InstanceSpec> builder = ImmutableList.builder();
+ for (int i = 0; i < instanceQty; ++i)
+ builder.add(InstanceSpec.newInstanceSpec());
+
+ return listToMap(builder.build());
+ }
+
+ private static Map<InstanceSpec, Collection<InstanceSpec>> listToMap(Collection<InstanceSpec> list) {
+ ImmutableMap.Builder<InstanceSpec, Collection<InstanceSpec>> mapBuilder = ImmutableMap.builder();
+ for (InstanceSpec spec : list)
+ mapBuilder.put(spec, list);
+
+ return mapBuilder.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/dfaffe2a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingZooKeeperServer.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingZooKeeperServer.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingZooKeeperServer.java
new file mode 100644
index 0000000..808e617
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingZooKeeperServer.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator;
+
+import org.apache.curator.test.DirectoryUtils;
+import org.apache.curator.test.InstanceSpec;
+import org.apache.curator.test.QuorumConfigBuilder;
+import org.apache.curator.test.TestingZooKeeperMain;
+import org.apache.curator.test.ZooKeeperMainFace;
+import org.apache.zookeeper.server.quorum.QuorumPeer;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ */
+public class TestingZooKeeperServer extends QuorumPeerMain implements Closeable {
+ /** Logger. */
+ private static final Logger logger = LoggerFactory.getLogger(org.apache.curator.test.TestingZooKeeperServer.class);
+
+ /** Config builder. */
+ private final QuorumConfigBuilder configBuilder;
+ /** This instance index. */
+ private final int thisInstanceIndex;
+ /** Main. */
+ private volatile ZooKeeperMainFace main;
+ /** State. */
+ private final AtomicReference<State> state = new AtomicReference<>(State.LATENT);
+
+ /**
+ * Server state.
+ */
+ private enum State {
+ /** Latent. */
+ LATENT,
+ /** Started. */
+ STARTED,
+ /** Stopped. */
+ STOPPED,
+ /** Closed. */
+ CLOSED
+ }
+
+ /**
+ * @param configBuilder Config builder.
+ */
+ public TestingZooKeeperServer(QuorumConfigBuilder configBuilder) {
+ this(configBuilder, 0);
+ }
+
+ /**
+ * @param configBuilder Config builder.
+ * @param thisInstanceIndex This instance index.
+ */
+ public TestingZooKeeperServer(QuorumConfigBuilder configBuilder, int thisInstanceIndex) {
+ this.configBuilder = configBuilder;
+ this.thisInstanceIndex = thisInstanceIndex;
+ main = (configBuilder.size() > 1) ? new FixedTestingQuorumPeerMain() : new TestingZooKeeperMain();
+ }
+
+ /** {@inheritDoc} */
+ public QuorumPeer getQuorumPeer() {
+ return main.getQuorumPeer();
+ }
+
+ /**
+ *
+ */
+ public Collection<InstanceSpec> getInstanceSpecs() {
+ return configBuilder.getInstanceSpecs();
+ }
+
+ public void kill() {
+ main.kill();
+ state.set(State.STOPPED);
+ }
+
+ /**
+ * Restart the server. If the server is running it will be stopped and then
+ * started again. If it is not running (in a LATENT or STOPPED state) then
+ * it will be restarted. If it is in a CLOSED state then an exception will
+ * be thrown.
+ *
+ * @throws Exception
+ */
+ public void restart() throws Exception {
+ // Can't restart from a closed state as all the temporary data is gone
+ if (state.get() == State.CLOSED)
+ throw new IllegalStateException("Cannot restart a closed instance");
+
+ // If the server's currently running then stop it.
+ if (state.get() == State.STARTED)
+ stop();
+
+ // Set to a LATENT state so we can restart
+ state.set(State.LATENT);
+
+ main = (configBuilder.size() > 1) ? new FixedTestingQuorumPeerMain() : new TestingZooKeeperMain();
+ start();
+ }
+
+ /**
+ *
+ */
+ public void stop() throws IOException {
+ if (state.compareAndSet(State.STARTED, State.STOPPED))
+ main.close();
+ }
+
+ /**
+ *
+ */
+ public InstanceSpec getInstanceSpec() {
+ return configBuilder.getInstanceSpec(thisInstanceIndex);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IOException {
+ stop();
+
+ if (state.compareAndSet(State.STOPPED, State.CLOSED)) {
+ InstanceSpec spec = getInstanceSpec();
+ if (spec.deleteDataDirectoryOnClose())
+ DirectoryUtils.deleteRecursively(spec.getDataDirectory());
+ }
+ }
+
+ /**
+ *
+ */
+ public void start() throws Exception {
+ if (!state.compareAndSet(State.LATENT, State.STARTED))
+ return;
+
+ new Thread(new Runnable() {
+ public void run() {
+ try {
+ QuorumPeerConfig config = configBuilder.buildConfig(thisInstanceIndex);
+ main.runFromConfig(config);
+ }
+ catch (Exception e) {
+ logger.error(String.format("From testing server (random state: %s) for instance: %s", String.valueOf(configBuilder.isFromRandom()), getInstanceSpec()), e);
+ }
+ }
+ }).start();
+
+ main.blockUntilStarted();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/dfaffe2a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
index c5d3488..9f0cc6d 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
@@ -22,7 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import junit.framework.TestSuite;
import org.apache.curator.test.InstanceSpec;
-import org.apache.curator.test.TestingCluster;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster;
import org.apache.ignite.IgniteException;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.spi.discovery.DiscoverySpi;
http://git-wip-us.apache.org/repos/asf/ignite/blob/dfaffe2a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
index 012366f..3b8ddee 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
@@ -18,7 +18,7 @@
package org.apache.ignite.spi.discovery.zk;
import junit.framework.TestSuite;
-import org.apache.curator.test.TestingCluster;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster;
import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest;
import org.apache.ignite.internal.IgniteClientReconnectCacheTest;
import org.apache.ignite.internal.processors.cache.datastructures.IgniteClientDataStructuresTest;
http://git-wip-us.apache.org/repos/asf/ignite/blob/dfaffe2a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
index 7c9ec51..e3b91c7 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
@@ -26,7 +26,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.curator.test.TestingCluster;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.typedef.internal.U;
http://git-wip-us.apache.org/repos/asf/ignite/blob/dfaffe2a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index 19ca5f2..f249b59 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -52,8 +52,8 @@ import javax.management.ObjectName;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.RetryNTimes;
-import org.apache.curator.test.TestingCluster;
-import org.apache.curator.test.TestingZooKeeperServer;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingZooKeeperServer;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
http://git-wip-us.apache.org/repos/asf/ignite/blob/dfaffe2a/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index 76e39b7..f5b49ba 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -122,7 +122,7 @@
<yammer.metrics.core.version>2.2.0</yammer.metrics.core.version>
<yardstick.version>0.8.3</yardstick.version>
<zkclient.version>0.5</zkclient.version>
- <zookeeper.version>3.4.6</zookeeper.version>
+ <zookeeper.version>3.4.13</zookeeper.version>
<!-- OSGI Manifest generation default property values -->
<osgi.import.package>*</osgi.import.package>
[08/50] [abbrv] ignite git commit: IGNITE-9373 Fixed MVCC failing
tests - Fixes #4623.
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java
index 57cee61..1a6cb55 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java
@@ -18,44 +18,30 @@
package org.apache.ignite.internal.processors.query.h2;
import java.io.File;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Semaphore;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.index.DynamicIndexAbstractSelfTest;
-import org.apache.ignite.internal.processors.cache.mvcc.MvccVersion;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
-import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
import org.apache.ignite.internal.processors.query.GridQueryProcessor;
-import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitor;
-import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitorImpl;
import org.apache.ignite.internal.util.lang.GridCursor;
-import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiTuple;
/**
* Index rebuild after node restart test.
*/
public class GridIndexRebuildSelfTest extends DynamicIndexAbstractSelfTest {
/** Data size. */
- private final static int AMOUNT = 10;
+ protected static final int AMOUNT = 300;
/** Data size. */
- private final static String CACHE_NAME = "T";
+ protected static final String CACHE_NAME = "T";
/** Test instance to allow interaction with static context. */
private static GridIndexRebuildSelfTest INSTANCE;
@@ -63,14 +49,13 @@ public class GridIndexRebuildSelfTest extends DynamicIndexAbstractSelfTest {
/** Latch to signal that rebuild may start. */
private final CountDownLatch rebuildLatch = new CountDownLatch(1);
- /** Latch to signal that concurrent put may start. */
- private final Semaphore rebuildSemaphore = new Semaphore(1, true);
-
/** {@inheritDoc} */
@Override protected IgniteConfiguration commonConfiguration(int idx) throws Exception {
IgniteConfiguration cfg = super.commonConfiguration(idx);
- cfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration().setPersistenceEnabled(true);
+ cfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration()
+ .setMaxSize(300*1024L*1024L)
+ .setPersistenceEnabled(true);
return cfg;
}
@@ -85,24 +70,25 @@ public class GridIndexRebuildSelfTest extends DynamicIndexAbstractSelfTest {
INSTANCE = this;
}
- /**
- * Do test with MVCC enabled.
- */
- public void testMvccEnabled() throws Exception {
- fail("https://issues.apache.org/jira/browse/IGNITE-7259");
- doTest(true);
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
}
- /**
- * Do test with MVCC disabled.
- */
- public void testMvccDisabled() throws Exception {
- fail("https://issues.apache.org/jira/browse/IGNITE-7259");
- doTest(false);
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ cleanPersistenceDir();
}
/**
- * Do test.<p>
+ * Do test.
+ * <p>
* Steps are as follows:
* <ul>
* <li>Put some data;</li>
@@ -120,12 +106,11 @@ public class GridIndexRebuildSelfTest extends DynamicIndexAbstractSelfTest {
* index rebuild for them has happened after put.</li>
* </ul>
* </li>
- * </ul>
- * @param mvccEnabled MVCC flag.
+ * </ul></p>
* @throws Exception if failed.
*/
- private void doTest(boolean mvccEnabled) throws Exception {
- IgniteEx srv = startServer(mvccEnabled);
+ public void testIndexRebuild() throws Exception {
+ IgniteEx srv = startServer();
execute(srv, "CREATE TABLE T(k int primary key, v int) WITH \"cache_name=T,wrap_value=false," +
"atomicity=transactional\"");
@@ -136,12 +121,9 @@ public class GridIndexRebuildSelfTest extends DynamicIndexAbstractSelfTest {
assertNotNull(cc);
- if (mvccEnabled)
- lockVersion(srv);
-
putData(srv, false);
- checkDataState(srv, mvccEnabled, false);
+ checkDataState(srv, false);
File cacheWorkDir = ((FilePageStoreManager)cc.context().shared().pageStore()).cacheWorkDir(cc.configuration());
@@ -151,22 +133,22 @@ public class GridIndexRebuildSelfTest extends DynamicIndexAbstractSelfTest {
assertTrue(U.delete(idxPath));
- srv = startServer(mvccEnabled);
+ srv = startServer();
putData(srv, true);
- checkDataState(srv, mvccEnabled, true);
+ checkDataState(srv, true);
}
/**
* Check versions presence in index tree.
+ *
* @param srv Node.
- * @param mvccEnabled MVCC flag.
* @param afterRebuild Whether index rebuild has occurred.
* @throws IgniteCheckedException if failed.
*/
@SuppressWarnings({"ConstantConditions", "unchecked"})
- private void checkDataState(IgniteEx srv, boolean mvccEnabled, boolean afterRebuild) throws IgniteCheckedException {
+ protected void checkDataState(IgniteEx srv, boolean afterRebuild) throws IgniteCheckedException {
IgniteInternalCache icache = srv.cachex(CACHE_NAME);
IgniteCache cache = srv.cache(CACHE_NAME);
@@ -179,47 +161,23 @@ public class GridIndexRebuildSelfTest extends DynamicIndexAbstractSelfTest {
while (cur.next()) {
CacheDataRow row = cur.get();
- int key = row.key().value(icache.context().cacheObjectContext(), false);
-
- if (mvccEnabled) {
- List<IgniteBiTuple<Object, MvccVersion>> vers = store.mvccFindAllVersions(icache.context(), row.key());
-
- if (!afterRebuild || key <= AMOUNT / 2)
- assertEquals(key, vers.size());
- else {
- // For keys affected by concurrent put there are two versions -
- // -1 (concurrent put mark) and newest restored value as long as put cleans obsolete versions.
- assertEquals(2, vers.size());
-
- assertEquals(-1, vers.get(0).getKey());
- assertEquals(key, vers.get(1).getKey());
- }
- }
- else {
- if (!afterRebuild || key <= AMOUNT / 2)
- assertEquals(key, cache.get(key));
- else
- assertEquals(-1, cache.get(key));
- }
+ int key = row.key().value(icache.context().cacheObjectContext(), false);
+
+ if (!afterRebuild || key <= AMOUNT / 2)
+ assertEquals(key, cache.get(key));
+ else
+ assertEquals(-1, cache.get(key));
}
}
}
/**
- * Lock coordinator version in order to keep MVCC versions in place.
- * @param node Node.
- * @throws IgniteCheckedException if failed.
- */
- private static void lockVersion(IgniteEx node) throws IgniteCheckedException {
- node.context().coordinators().requestSnapshotAsync().get();
- }
-
- /**
* Put data to cache.
+ *
* @param node Node.
* @throws Exception if failed.
*/
- private void putData(Ignite node, final boolean forConcurrentPut) throws Exception {
+ protected void putData(Ignite node, final boolean forConcurrentPut) throws Exception {
final IgniteCache<Integer, Integer> cache = node.cache(CACHE_NAME);
assertNotNull(cache);
@@ -230,13 +188,9 @@ public class GridIndexRebuildSelfTest extends DynamicIndexAbstractSelfTest {
if (i <= AMOUNT / 2)
continue;
- rebuildSemaphore.acquire();
-
cache.put(i, -1);
rebuildLatch.countDown();
-
- rebuildSemaphore.release();
}
else {
// Data streamer is not used intentionally in order to preserve all versions.
@@ -248,16 +202,16 @@ public class GridIndexRebuildSelfTest extends DynamicIndexAbstractSelfTest {
/**
* Start server node.
- * @param mvccEnabled MVCC flag.
+ *
* @return Started node.
* @throws Exception if failed.
*/
- private IgniteEx startServer(boolean mvccEnabled) throws Exception {
+ protected IgniteEx startServer() throws Exception {
// Have to do this for each starting node - see GridQueryProcessor ctor, it nulls
// idxCls static field on each call.
GridQueryProcessor.idxCls = BlockingIndexing.class;
- IgniteConfiguration cfg = serverConfiguration(0).setMvccEnabled(mvccEnabled);
+ IgniteConfiguration cfg = serverConfiguration(0);
IgniteEx res = startGrid(cfg);
@@ -266,21 +220,6 @@ public class GridIndexRebuildSelfTest extends DynamicIndexAbstractSelfTest {
return res;
}
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
-
- stopAllGrids();
-
- cleanPersistenceDir();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- super.afterTestsStopped();
-
- cleanPersistenceDir();
- }
-
/**
* Blocking indexing processor.
*/
@@ -295,56 +234,7 @@ public class GridIndexRebuildSelfTest extends DynamicIndexAbstractSelfTest {
else
firstRbld = false;
- int cacheId = CU.cacheId(cacheName);
-
- GridCacheContext cctx = ctx.cache().context().cacheContext(cacheId);
-
- final GridCacheQueryManager qryMgr = cctx.queries();
-
- SchemaIndexCacheVisitor visitor = new SchemaIndexCacheVisitorImpl(cctx);
-
- visitor.visit(new TestRebuildClosure(qryMgr, cctx.mvccEnabled()));
-
- for (H2TableDescriptor tblDesc : tables(cacheName))
- tblDesc.table().markRebuildFromHashInProgress(false);
- }
- }
-
- /**
- * Test closure.
- */
- private final static class TestRebuildClosure extends RebuildIndexFromHashClosure {
- /** Seen keys set to track moment when concurrent put may start. */
- private final Set<KeyCacheObject> keys =
- Collections.newSetFromMap(new ConcurrentHashMap<KeyCacheObject, Boolean>());
-
- /**
- * @param qryMgr Query manager.
- * @param mvccEnabled MVCC status flag.
- */
- TestRebuildClosure(GridCacheQueryManager qryMgr, boolean mvccEnabled) {
- super(qryMgr, mvccEnabled);
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void apply(CacheDataRow row) throws IgniteCheckedException {
- // For half of the keys, we want to do rebuild
- // after corresponding key had been put from a concurrent thread.
- boolean keyFirstMet = keys.add(row.key()) && keys.size() > AMOUNT / 2;
-
- if (keyFirstMet) {
- try {
- INSTANCE.rebuildSemaphore.acquire();
- }
- catch (InterruptedException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- super.apply(row);
-
- if (keyFirstMet)
- INSTANCE.rebuildSemaphore.release();
+ super.rebuildIndexesFromHash(cacheName);
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildWithMvccEnabledSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildWithMvccEnabledSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildWithMvccEnabledSelfTest.java
new file mode 100644
index 0000000..3d65db8
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildWithMvccEnabledSelfTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2;
+
+import java.io.File;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheObjectContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.mvcc.MvccVersion;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.util.lang.GridCursor;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+
+/**
+ * Index rebuild after node restart test.
+ */
+public class GridIndexRebuildWithMvccEnabledSelfTest extends GridIndexRebuildSelfTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration serverConfiguration(int idx, boolean filter) throws Exception {
+ return super.serverConfiguration(idx, filter)
+ .setMvccVacuumTimeInterval(Integer.MAX_VALUE)
+ .setMvccEnabled(true);
+ }
+
+ /** {@inheritDoc} */
+ public void testIndexRebuild() throws Exception {
+ IgniteEx srv = startServer();
+
+ execute(srv, "CREATE TABLE T(k int primary key, v int) WITH \"cache_name=T,wrap_value=false," +
+ "atomicity=transactional\"");
+
+ execute(srv, "CREATE INDEX IDX ON T(v)");
+
+ IgniteInternalCache cc = srv.cachex(CACHE_NAME);
+
+ assertNotNull(cc);
+
+ lockVersion(srv);
+
+ putData(srv, false);
+
+ checkDataState(srv, false);
+
+ File cacheWorkDir = ((FilePageStoreManager)cc.context().shared().pageStore()).cacheWorkDir(cc.configuration());
+
+ File idxPath = cacheWorkDir.toPath().resolve("index.bin").toFile();
+
+ stopAllGrids();
+
+ assertTrue(U.delete(idxPath));
+
+ srv = startServer();
+
+ putData(srv, true);
+
+ checkDataState(srv, true);
+ }
+
+ /**
+ * Lock coordinator version in order to keep MVCC versions in place.
+ *
+ * @param node Node.
+ * @throws IgniteCheckedException if failed.
+ */
+ private static void lockVersion(IgniteEx node) throws IgniteCheckedException {
+ node.context().coordinators().requestSnapshotAsync().get();
+ }
+
+ /** {@inheritDoc} */
+ protected void checkDataState(IgniteEx srv, boolean afterRebuild) throws IgniteCheckedException {
+ IgniteInternalCache icache = srv.cachex(CACHE_NAME);
+
+ assertNotNull(icache);
+
+ CacheObjectContext coCtx = icache.context().cacheObjectContext();
+
+ for (IgniteCacheOffheapManager.CacheDataStore store : icache.context().offheap().cacheDataStores()) {
+ GridCursor<? extends CacheDataRow> cur = store.cursor();
+
+ while (cur.next()) {
+ CacheDataRow row = cur.get();
+
+ int key = row.key().value(coCtx, false);
+
+ List<IgniteBiTuple<Object, MvccVersion>> vers = store.mvccFindAllVersions(icache.context(), row.key());
+
+ if (!afterRebuild || key <= AMOUNT / 2)
+ assertEquals(key, vers.size());
+ else {
+ // For keys affected by concurrent put there are two versions -
+ // -1 (concurrent put mark) and newest restored value as long as put cleans obsolete versions.
+ assertEquals(2, vers.size());
+
+ Object val0 = ((CacheObject)vers.get(0).getKey()).value(coCtx, false);
+ Object val1 = ((CacheObject)vers.get(1).getKey()).value(coCtx, false);
+
+ assertEquals(-1, val0);
+ assertEquals(key, val1);
+ }
+
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java
index a8087da..ab9d2e6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java
@@ -18,6 +18,7 @@
package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.index.SqlTransactionsCommandsWithMvccEnabledSelfTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccBulkLoadTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccDmlSimpleTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccIteratorWithConcurrentJdbcTransactionTest;
@@ -37,6 +38,7 @@ import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccReplicatedSqlTx
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccScanQueryWithConcurrentJdbcTransactionTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccSizeWithConcurrentJdbcTransactionTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccStreamingInsertTest;
+import org.apache.ignite.internal.processors.query.h2.GridIndexRebuildWithMvccEnabledSelfTest;
/**
*
@@ -48,25 +50,39 @@ public class IgniteCacheMvccSqlTestSuite extends TestSuite {
public static TestSuite suite() {
TestSuite suite = new TestSuite("IgniteCache SQL MVCC Test Suite");
+ // Simle tests.
+ suite.addTestSuite(CacheMvccDmlSimpleTest.class);
+ suite.addTestSuite(SqlTransactionsCommandsWithMvccEnabledSelfTest.class);
+
+ suite.addTestSuite(GridIndexRebuildWithMvccEnabledSelfTest.class);
+
+ // JDBC tests.
suite.addTestSuite(CacheMvccSizeWithConcurrentJdbcTransactionTest.class);
suite.addTestSuite(CacheMvccScanQueryWithConcurrentJdbcTransactionTest.class);
suite.addTestSuite(CacheMvccLocalEntriesWithConcurrentJdbcTransactionTest.class);
suite.addTestSuite(CacheMvccIteratorWithConcurrentJdbcTransactionTest.class);
+
+ // Load tests.
+ suite.addTestSuite(CacheMvccBulkLoadTest.class);
+ suite.addTestSuite(CacheMvccStreamingInsertTest.class);
+
suite.addTestSuite(CacheMvccPartitionedSqlQueriesTest.class);
suite.addTestSuite(CacheMvccReplicatedSqlQueriesTest.class);
suite.addTestSuite(CacheMvccPartitionedSqlTxQueriesTest.class);
suite.addTestSuite(CacheMvccReplicatedSqlTxQueriesTest.class);
+
suite.addTestSuite(CacheMvccPartitionedSqlTxQueriesWithReducerTest.class);
suite.addTestSuite(CacheMvccReplicatedSqlTxQueriesWithReducerTest.class);
suite.addTestSuite(CacheMvccPartitionedSelectForUpdateQueryTest.class);
suite.addTestSuite(CacheMvccReplicatedSelectForUpdateQueryTest.class);
+
+ // Failover tests.
suite.addTestSuite(CacheMvccPartitionedBackupsTest.class);
suite.addTestSuite(CacheMvccReplicatedBackupsTest.class);
+
suite.addTestSuite(CacheMvccPartitionedSqlCoordinatorFailoverTest.class);
suite.addTestSuite(CacheMvccReplicatedSqlCoordinatorFailoverTest.class);
- suite.addTestSuite(CacheMvccBulkLoadTest.class);
- suite.addTestSuite(CacheMvccStreamingInsertTest.class);
- suite.addTestSuite(CacheMvccDmlSimpleTest.class);
+
return suite;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index b0670e8..671db13 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -110,7 +110,6 @@ import org.apache.ignite.internal.processors.cache.index.DynamicIndexServerCoord
import org.apache.ignite.internal.processors.cache.index.DynamicIndexServerNodeFIlterBasicSelfTest;
import org.apache.ignite.internal.processors.cache.index.DynamicIndexServerNodeFilterCoordinatorBasicSelfTest;
import org.apache.ignite.internal.processors.cache.index.H2ConnectionLeaksSelfTest;
-import org.apache.ignite.internal.processors.cache.index.IgniteDecimalSelfTest;
import org.apache.ignite.internal.processors.cache.index.H2DynamicColumnsClientBasicSelfTest;
import org.apache.ignite.internal.processors.cache.index.H2DynamicColumnsServerBasicSelfTest;
import org.apache.ignite.internal.processors.cache.index.H2DynamicColumnsServerCoordinatorBasicSelfTest;
@@ -131,11 +130,11 @@ import org.apache.ignite.internal.processors.cache.index.H2DynamicIndexingComple
import org.apache.ignite.internal.processors.cache.index.H2DynamicTableSelfTest;
import org.apache.ignite.internal.processors.cache.index.H2RowCachePageEvictionTest;
import org.apache.ignite.internal.processors.cache.index.H2RowCacheSelfTest;
+import org.apache.ignite.internal.processors.cache.index.IgniteDecimalSelfTest;
import org.apache.ignite.internal.processors.cache.index.LongIndexNameTest;
import org.apache.ignite.internal.processors.cache.index.OptimizedMarshallerIndexNameTest;
import org.apache.ignite.internal.processors.cache.index.SchemaExchangeSelfTest;
-import org.apache.ignite.internal.processors.cache.index.SqlTransactionsComandsWithMvccDisabledSelfTest;
-import org.apache.ignite.internal.processors.cache.index.SqlTransactionsSelfTest;
+import org.apache.ignite.internal.processors.cache.index.SqlTransactionsComandsSelfTest;
import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalAtomicQuerySelfTest;
import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalFieldsQuerySelfTest;
import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalQueryCancelOrTimeoutSelfTest;
@@ -165,18 +164,18 @@ import org.apache.ignite.internal.processors.query.IgniteSqlSplitterSelfTest;
import org.apache.ignite.internal.processors.query.LazyQuerySelfTest;
import org.apache.ignite.internal.processors.query.MultipleStatementsSqlQuerySelfTest;
import org.apache.ignite.internal.processors.query.SqlIllegalSchemaSelfTest;
-import org.apache.ignite.internal.processors.query.SqlSystemViewsSelfTest;
import org.apache.ignite.internal.processors.query.SqlPushDownFunctionTest;
import org.apache.ignite.internal.processors.query.SqlSchemaSelfTest;
+import org.apache.ignite.internal.processors.query.SqlSystemViewsSelfTest;
import org.apache.ignite.internal.processors.query.h2.GridH2IndexingInMemSelfTest;
import org.apache.ignite.internal.processors.query.h2.GridH2IndexingOffheapSelfTest;
import org.apache.ignite.internal.processors.query.h2.GridIndexRebuildSelfTest;
-import org.apache.ignite.internal.processors.query.h2.H2StatementCacheSelfTest;
import org.apache.ignite.internal.processors.query.h2.H2ResultSetIteratorNullifyOnEndSelfTest;
+import org.apache.ignite.internal.processors.query.h2.H2StatementCacheSelfTest;
import org.apache.ignite.internal.processors.query.h2.IgniteSqlBigIntegerKeyTest;
import org.apache.ignite.internal.processors.query.h2.IgniteSqlQueryMinMaxTest;
-import org.apache.ignite.internal.processors.query.h2.ThreadLocalObjectPoolSelfTest;
import org.apache.ignite.internal.processors.query.h2.PreparedStatementExSelfTest;
+import org.apache.ignite.internal.processors.query.h2.ThreadLocalObjectPoolSelfTest;
import org.apache.ignite.internal.processors.query.h2.sql.BaseH2CompareQueryTest;
import org.apache.ignite.internal.processors.query.h2.sql.GridQueryParsingTest;
import org.apache.ignite.internal.processors.query.h2.sql.H2CompareBigQueryDistributedJoinsTest;
@@ -447,8 +446,7 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
suite.addTestSuite(GridIndexRebuildSelfTest.class);
- suite.addTestSuite(SqlTransactionsSelfTest.class);
- suite.addTestSuite(SqlTransactionsComandsWithMvccDisabledSelfTest.class);
+ suite.addTestSuite(SqlTransactionsComandsSelfTest.class);
suite.addTestSuite(IgniteSqlDefaultValueTest.class);
suite.addTestSuite(IgniteDecimalSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
index 1fd8e72..1f600dd 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
@@ -80,7 +80,10 @@ namespace Apache.Ignite.Core.Tests.ApiParity
"TimeServerPortBase",
"TimeServerPortRange",
"IncludeProperties",
- "isAutoActivationEnabled" // IGNITE-7301
+ "isAutoActivationEnabled", // IGNITE-7301
+ "isMvccEnabled", //TODO: IGNITE-9390: Remove when Mvcc support will be added.
+ "MvccVacuumTimeInterval", //TODO: IGNITE-9390: Remove when Mvcc support will be added.
+ "MvccVacuumThreadCnt" //TODO: IGNITE-9390: Remove when Mvcc support will be added.
};
/// <summary>
[09/50] [abbrv] ignite git commit: IGNITE-9373 Fixed MVCC failing
tests - Fixes #4623.
Posted by ag...@apache.org.
IGNITE-9373 Fixed MVCC failing tests - Fixes #4623.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/254a6529
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/254a6529
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/254a6529
Branch: refs/heads/ignite-5960
Commit: 254a6529228240dcc229a875b2f2847d456b5610
Parents: 4d736fc
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Fri Aug 31 19:24:00 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Aug 31 19:24:00 2018 +0300
----------------------------------------------------------------------
.../suite/IgniteJdbcDriverMvccTestSuite.java | 47 ++
.../jdbc/suite/IgniteJdbcDriverTestSuite.java | 1 -
...ThinTransactionsWithMvccEnabledSelfTest.java | 447 +++++++++++++++++++
.../cache/IgniteCacheOffheapManagerImpl.java | 11 +-
.../cache/mvcc/MvccProcessorImpl.java | 35 +-
.../processors/cache/mvcc/MvccUtils.java | 11 +-
.../cache/query/GridCacheQueryAdapter.java | 16 +-
...vccAbstractBasicCoordinatorFailoverTest.java | 4 -
.../mvcc/CacheMvccAbstractFeatureTest.java | 13 +-
.../cache/mvcc/CacheMvccAbstractTest.java | 21 +-
.../cache/mvcc/CacheMvccClusterRestartTest.java | 18 +-
.../cache/mvcc/CacheMvccTransactionsTest.java | 37 +-
.../DataStreamProcessorMvccSelfTest.java | 70 +++
.../junits/common/GridCommonAbstractTest.java | 11 +-
.../testsuites/IgniteCacheMvccTestSuite.java | 14 +-
.../processors/query/h2/H2FieldsIterator.java | 2 +-
.../query/h2/H2ResultSetIterator.java | 25 +-
.../index/SqlTransactionsComandsSelfTest.java | 83 ++++
...sactionsCommandsWithMvccEnabledSelfTest.java | 420 +++++++++++++++++
.../mvcc/CacheMvccBackupsAbstractTest.java | 6 +-
.../mvcc/CacheMvccSqlQueriesAbstractTest.java | 2 +
...MvccSqlTxQueriesWithReducerAbstractTest.java | 2 +-
.../query/h2/GridIndexRebuildSelfTest.java | 188 ++------
...GridIndexRebuildWithMvccEnabledSelfTest.java | 126 ++++++
.../testsuites/IgniteCacheMvccSqlTestSuite.java | 22 +-
.../IgniteCacheQuerySelfTestSuite.java | 14 +-
.../ApiParity/IgniteConfigurationParityTest.cs | 5 +-
27 files changed, 1400 insertions(+), 251 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverMvccTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverMvccTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverMvccTestSuite.java
new file mode 100644
index 0000000..6d8933d
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverMvccTestSuite.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc.suite;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.jdbc.thin.JdbcThinConnectionMvccEnabledSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinTransactionsClientAutoCommitComplexSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinTransactionsClientNoAutoCommitComplexSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinTransactionsWithMvccEnabledSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinTransactionsServerAutoCommitComplexSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinTransactionsServerNoAutoCommitComplexSelfTest;
+
+public class IgniteJdbcDriverMvccTestSuite extends TestSuite {
+ /**
+ * @return JDBC Driver Test Suite.
+ * @throws Exception In case of error.
+ */
+ public static TestSuite suite() throws Exception {
+ TestSuite suite = new TestSuite("Ignite JDBC Driver Test Suite");
+
+ suite.addTest(new TestSuite(JdbcThinConnectionMvccEnabledSelfTest.class));
+
+ // Transactions
+ suite.addTest(new TestSuite(JdbcThinTransactionsWithMvccEnabledSelfTest.class));
+ suite.addTest(new TestSuite(JdbcThinTransactionsClientAutoCommitComplexSelfTest.class));
+ suite.addTest(new TestSuite(JdbcThinTransactionsServerAutoCommitComplexSelfTest.class));
+ suite.addTest(new TestSuite(JdbcThinTransactionsClientNoAutoCommitComplexSelfTest.class));
+ suite.addTest(new TestSuite(JdbcThinTransactionsServerNoAutoCommitComplexSelfTest.class));
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index 275040f..7fbf41f 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -19,7 +19,6 @@ package org.apache.ignite.jdbc.suite;
import junit.framework.TestSuite;
import org.apache.ignite.internal.jdbc2.JdbcBlobTest;
-import org.apache.ignite.internal.jdbc2.JdbcBulkLoadSelfTest;
import org.apache.ignite.internal.jdbc2.JdbcConnectionReopenTest;
import org.apache.ignite.internal.jdbc2.JdbcDistributedJoinsQueryTest;
import org.apache.ignite.jdbc.JdbcComplexQuerySelfTest;
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinTransactionsWithMvccEnabledSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinTransactionsWithMvccEnabledSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinTransactionsWithMvccEnabledSelfTest.java
new file mode 100644
index 0000000..e3f7f14
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinTransactionsWithMvccEnabledSelfTest.java
@@ -0,0 +1,447 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc.thin;
+
+import java.sql.BatchUpdateException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.internal.processors.query.NestedTxMode;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridStringLogger;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Tests to check behavior with transactions on.
+ */
+public class JdbcThinTransactionsWithMvccEnabledSelfTest extends JdbcThinAbstractSelfTest {
+ /** IP finder. */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final String URL = "jdbc:ignite:thin://127.0.0.1";
+
+ /** Logger. */
+ private GridStringLogger log;
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("deprecation")
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setMvccEnabled(true);
+
+ cfg.setCacheConfiguration(cacheConfiguration(DEFAULT_CACHE_NAME));
+
+ TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+ disco.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(disco);
+
+ cfg.setMarshaller(new BinaryMarshaller());
+
+ cfg.setGridLogger(log = new GridStringLogger());
+
+ return cfg;
+ }
+
+ /**
+ * @param name Cache name.
+ * @return Cache configuration.
+ * @throws Exception In case of error.
+ */
+ private CacheConfiguration cacheConfiguration(@NotNull String name) throws Exception {
+ CacheConfiguration cfg = defaultCacheConfiguration();
+
+ cfg.setName(name);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ startGrid(0);
+
+ try (Connection c = c(true, NestedTxMode.ERROR)) {
+ try (Statement s = c.createStatement()) {
+ s.execute("CREATE TABLE INTS (k int primary key, v int) WITH \"cache_name=ints,wrap_value=false," +
+ "atomicity=transactional\"");
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ * @param autoCommit Auto commit mode.
+ * @param nestedTxMode Nested transactions mode.
+ * @return Connection.
+ * @throws SQLException if failed.
+ */
+ private static Connection c(boolean autoCommit, NestedTxMode nestedTxMode) throws SQLException {
+ Connection res = DriverManager.getConnection(URL + "/?nestedTransactionsMode=" + nestedTxMode.name());
+
+ res.setAutoCommit(autoCommit);
+
+ return res;
+ }
+
+ /**
+ *
+ */
+ public void testTransactionsBeginCommitRollback() throws IgniteCheckedException {
+ final AtomicBoolean stop = new AtomicBoolean();
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ try (Connection c = c(false, NestedTxMode.ERROR)) {
+ while (!stop.get()) {
+ try (Statement s = c.createStatement()) {
+ s.execute("BEGIN");
+
+ c.commit();
+
+ s.execute("BEGIN");
+
+ c.rollback();
+ }
+ }
+ }
+ }
+ catch (SQLException e) {
+ throw new AssertionError(e);
+ }
+ }
+ }, 8, "jdbc-transactions");
+
+ U.sleep(5000);
+
+ stop.set(true);
+
+ fut.get();
+ }
+
+ /**
+ *
+ */
+ public void testTransactionsBeginCommitRollbackAutocommit() throws IgniteCheckedException {
+ GridTestUtils.runMultiThreadedAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ try (Connection c = c(true, NestedTxMode.ERROR)) {
+ try (Statement s = c.createStatement()) {
+ s.execute("BEGIN");
+
+ s.execute("COMMIT");
+
+ s.execute("BEGIN");
+
+ s.execute("ROLLBACK");
+ }
+ }
+ }
+ catch (SQLException e) {
+ throw new AssertionError(e);
+ }
+ }
+ }, 8, "jdbc-transactions").get();
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testIgnoreNestedTxAutocommitOff() throws SQLException {
+ try (Connection c = c(false, NestedTxMode.IGNORE)) {
+ doNestedTxStart(c, false);
+ }
+
+ assertTrue(log.toString().contains("ignoring BEGIN command"));
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testCommitNestedTxAutocommitOff() throws SQLException {
+ try (Connection c = c(false, NestedTxMode.COMMIT)) {
+ doNestedTxStart(c, false);
+ }
+
+ assertFalse(log.toString().contains("ignoring BEGIN command"));
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testErrorNestedTxAutocommitOff() throws SQLException {
+ GridTestUtils.assertThrows(null, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try (Connection c = c(false, NestedTxMode.ERROR)) {
+ doNestedTxStart(c, false);
+ }
+
+ throw new AssertionError();
+ }
+ }, SQLException.class, "Transaction has already been started.");
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testIgnoreNestedTxAutocommitOn() throws SQLException {
+ try (Connection c = c(true, NestedTxMode.IGNORE)) {
+ doNestedTxStart(c, false);
+ }
+
+ assertTrue(log.toString().contains("ignoring BEGIN command"));
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testCommitNestedTxAutocommitOn() throws SQLException {
+ try (Connection c = c(true, NestedTxMode.COMMIT)) {
+ doNestedTxStart(c, false);
+ }
+
+ assertFalse(log.toString().contains("ignoring BEGIN command"));
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testErrorNestedTxAutocommitOn() throws SQLException {
+ GridTestUtils.assertThrows(null, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try (Connection c = c(true, NestedTxMode.ERROR)) {
+ doNestedTxStart(c, false);
+ }
+
+ throw new AssertionError();
+ }
+ }, SQLException.class, "Transaction has already been started.");
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testIgnoreNestedTxAutocommitOffBatched() throws SQLException {
+ try (Connection c = c(false, NestedTxMode.IGNORE)) {
+ doNestedTxStart(c, true);
+ }
+
+ assertTrue(log.toString().contains("ignoring BEGIN command"));
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testCommitNestedTxAutocommitOffBatched() throws SQLException {
+ try (Connection c = c(false, NestedTxMode.COMMIT)) {
+ doNestedTxStart(c, true);
+ }
+
+ assertFalse(log.toString().contains("ignoring BEGIN command"));
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testErrorNestedTxAutocommitOffBatched() throws SQLException {
+ GridTestUtils.assertThrows(null, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try (Connection c = c(false, NestedTxMode.ERROR)) {
+ doNestedTxStart(c, true);
+ }
+
+ throw new AssertionError();
+ }
+ }, BatchUpdateException.class, "Transaction has already been started.");
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testIgnoreNestedTxAutocommitOnBatched() throws SQLException {
+ try (Connection c = c(true, NestedTxMode.IGNORE)) {
+ doNestedTxStart(c, true);
+ }
+
+ assertTrue(log.toString().contains("ignoring BEGIN command"));
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testCommitNestedTxAutocommitOnBatched() throws SQLException {
+ try (Connection c = c(true, NestedTxMode.COMMIT)) {
+ doNestedTxStart(c, true);
+ }
+
+ assertFalse(log.toString().contains("ignoring BEGIN command"));
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testErrorNestedTxAutocommitOnBatched() throws SQLException {
+ GridTestUtils.assertThrows(null, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try (Connection c = c(true, NestedTxMode.ERROR)) {
+ doNestedTxStart(c, true);
+ }
+
+ throw new AssertionError();
+ }
+ }, BatchUpdateException.class, "Transaction has already been started.");
+ }
+
+ /**
+ * Try to start nested transaction via batch as well as separate statements.
+ * @param conn Connection.
+ * @param batched Whether {@link Statement#executeBatch()} should be used.
+ * @throws SQLException if failed.
+ */
+ private void doNestedTxStart(Connection conn, boolean batched) throws SQLException {
+ try (Statement s = conn.createStatement()) {
+ s.executeQuery("SELECT * FROM INTS");
+
+ if (batched) {
+ s.addBatch("BEGIN");
+
+ s.addBatch("BEGIN");
+
+ s.executeBatch();
+ }
+ else {
+ s.execute("BEGIN");
+
+ s.execute("BEGIN");
+ }
+ }
+ }
+
+ /**
+ * @throws SQLException if failed.
+ */
+ public void testAutoCommitSingle() throws SQLException {
+ doTestAutoCommit(false);
+ }
+
+ /**
+ * @throws SQLException if failed.
+ */
+ public void testAutoCommitBatched() throws SQLException {
+ doTestAutoCommit(true);
+ }
+
+ /**
+ * @param batched Batch mode flag.
+ * @throws SQLException if failed.
+ */
+ private void doTestAutoCommit(boolean batched) throws SQLException {
+ IgniteCache<Integer, ?> cache = grid(0).cache("ints");
+
+ try (Connection c = c(false, NestedTxMode.ERROR)) {
+ try (Statement s = c.createStatement()) {
+ assertFalse(s.executeQuery("SELECT * from INTS").next());
+
+ if (batched) {
+ s.addBatch("INSERT INTO INTS(k, v) values(1, 1)");
+
+ s.executeBatch();
+ }
+ else
+ s.execute("INSERT INTO INTS(k, v) values(1, 1)");
+
+ // We haven't committed anything yet - this check shows that autoCommit flag is in effect.
+ assertTrue(cache.query(new SqlFieldsQuery("SELECT * from INTS")).getAll().isEmpty());
+
+ // We should see own updates.
+ assertTrue(s.executeQuery("SELECT * from INTS").next());
+
+ c.commit();
+
+ c.setAutoCommit(true);
+
+ assertEquals(1, cache.get(1));
+
+ assertTrue(s.executeQuery("SELECT * from INTS").next());
+ }
+ }
+ }
+
+ /**
+ * Test that exception in one of the statements does not kill connection worker altogether.
+ * @throws SQLException if failed.
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testExceptionHandling() throws SQLException {
+ try (Connection c = c(true, NestedTxMode.ERROR)) {
+ try (Statement s = c.createStatement()) {
+ s.execute("INSERT INTO INTS(k, v) values(1, 1)");
+
+ assertEquals(1, grid(0).cache("ints").get(1));
+
+ GridTestUtils.assertThrows(null, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ s.execute("INSERT INTO INTS(x, y) values(1, 1)");
+
+ return null;
+ }
+ }, SQLException.class, "Failed to parse query");
+
+ s.execute("INSERT INTO INTS(k, v) values(2, 2)");
+
+ assertEquals(2, grid(0).cache("ints").get(2));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index bc35264..13ad7e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -2565,7 +2565,9 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
List<IgniteBiTuple<Object, MvccVersion>> res = new ArrayList<>();
- long crd = MVCC_CRD_COUNTER_NA, cntr = MVCC_COUNTER_NA; int opCntr = MVCC_OP_COUNTER_NA;
+ long crd = MVCC_CRD_COUNTER_NA;
+ long cntr = MVCC_COUNTER_NA;
+ int opCntr = MVCC_OP_COUNTER_NA;
while (cur.next()) {
CacheDataRow row = cur.get();
@@ -2575,7 +2577,9 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
res.add(F.t(row.value(), row.mvccVersion()));
- crd = row.mvccCoordinatorVersion(); cntr = row.mvccCounter(); opCntr = row.mvccOperationCounter();
+ crd = row.mvccCoordinatorVersion();
+ cntr = row.mvccCounter();
+ opCntr = row.mvccOperationCounter();
}
return res;
@@ -2654,8 +2658,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
return cursor(cacheId, null, null);
}
- /** {@inheritDoc}
- * @param cacheId*/
+ /** {@inheritDoc} */
@Override public GridCursor<? extends CacheDataRow> cursor(int cacheId,
MvccSnapshot mvccSnapshot) throws IgniteCheckedException {
return cursor(cacheId, null, null, null, mvccSnapshot);
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java
index 31d3b61..220f0c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java
@@ -545,7 +545,7 @@ class MvccProcessorImpl extends GridProcessorAdapter implements MvccProcessor, D
}
if (!ctx.localNodeId().equals(crd.nodeId()) || !initFut.isDone())
- return null;
+ return null;
else if (tx != null)
return assignTxSnapshot(0L);
else
@@ -1324,7 +1324,9 @@ class MvccProcessorImpl extends GridProcessorAdapter implements MvccProcessor, D
* @param msg Message.
*/
private void processCoordinatorSnapshotResponse(UUID nodeId, MvccSnapshotResponse msg) {
- Map<Long, MvccSnapshotResponseListener> map = snapLsnrs.get(nodeId); MvccSnapshotResponseListener lsnr;
+ Map<Long, MvccSnapshotResponseListener> map = snapLsnrs.get(nodeId);
+
+ MvccSnapshotResponseListener lsnr;
if (map != null && (lsnr = map.remove(msg.futureId())) != null)
lsnr.onResponse(msg);
@@ -2007,18 +2009,22 @@ class MvccProcessorImpl extends GridProcessorAdapter implements MvccProcessor, D
try {
GridCursor<? extends CacheDataRow> cursor = part.dataStore().cursor(KEY_ONLY);
- KeyCacheObject prevKey = null; Object rest = null;
+ KeyCacheObject prevKey = null;
+
+ Object rest = null;
List<MvccLinkAwareSearchRow> cleanupRows = null;
MvccSnapshot snapshot = task.snapshot();
- GridCacheContext cctx = null; int curCacheId = CU.UNDEFINED_CACHE_ID;
+ GridCacheContext cctx = null;
+
+ int curCacheId = CU.UNDEFINED_CACHE_ID;
boolean shared = part.group().sharedGroup();
- if (!shared)
- cctx = part.group().singleCacheContext();
+ if (!shared && (cctx = F.first(part.group().caches())) == null)
+ return metrics;
while (cursor.next()) {
if (isCancelled())
@@ -2032,19 +2038,27 @@ class MvccProcessorImpl extends GridProcessorAdapter implements MvccProcessor, D
if (cctx == null) {
assert shared;
- curCacheId = row.cacheId();
- cctx = part.group().shared().cacheContext(curCacheId);
+ cctx = part.group().shared().cacheContext(curCacheId = row.cacheId());
+
+ if (cctx == null)
+ return metrics;
}
if (!prevKey.equals(row.key()) || (shared && curCacheId != row.cacheId())) {
if (rest != null || !F.isEmpty(cleanupRows))
cleanup(part, prevKey, cleanupRows, rest, cctx, metrics);
- cleanupRows = null; rest = null;
+ cleanupRows = null;
- if (shared && curCacheId != row.cacheId())
+ rest = null;
+
+ if (shared && curCacheId != row.cacheId()) {
cctx = part.group().shared().cacheContext(curCacheId = row.cacheId());
+ if (cctx == null)
+ return metrics;
+ }
+
prevKey = row.key();
}
@@ -2127,7 +2141,6 @@ class MvccProcessorImpl extends GridProcessorAdapter implements MvccProcessor, D
}
/**
- *
* @param part Local partition.
* @param key Key.
* @param cleanupRows Cleanup rows.
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java
index 33f457d..c75393e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccUtils.java
@@ -758,12 +758,7 @@ public class MvccUtils {
if (tx == null)
tracker = new MvccQueryTrackerImpl(cctx);
else if ((tracker = tx.mvccQueryTracker()) == null)
- tracker = new StaticMvccQueryTracker(cctx, requestSnapshot(cctx, tx)) {
- @Override public void onDone() {
- // TODO IGNITE-8841
- checkActive(tx);
- }
- };
+ tracker = new StaticMvccQueryTracker(cctx, requestSnapshot(cctx, tx));
if (tracker.snapshot() == null)
// TODO IGNITE-7388
@@ -780,7 +775,9 @@ public class MvccUtils {
*/
public static MvccSnapshot requestSnapshot(GridCacheContext cctx,
GridNearTxLocal tx) throws IgniteCheckedException {
- MvccSnapshot snapshot; tx = checkActive(tx);
+ MvccSnapshot snapshot;
+
+ tx = checkActive(tx);
if ((snapshot = tx.mvccSnapshot()) == null) {
MvccProcessor prc = cctx.shared().coordinators();
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
index f21a22f..07aea4c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
@@ -45,6 +45,7 @@ import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtUnreservedPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
import org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker;
import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
import org.apache.ignite.internal.processors.cache.mvcc.MvccUtils;
@@ -540,8 +541,19 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
MvccQueryTracker mvccTracker = null;
- if (cctx.mvccEnabled() && mvccSnapshot == null)
- mvccSnapshot = (mvccTracker = MvccUtils.mvccTracker(cctx, false)).snapshot();
+ if (cctx.mvccEnabled() && mvccSnapshot == null) {
+ GridNearTxLocal tx = cctx.tm().userTx();
+
+ if (tx != null)
+ mvccSnapshot = MvccUtils.requestSnapshot(cctx, tx);
+ else {
+ mvccTracker = MvccUtils.mvccTracker(cctx, null);
+
+ mvccSnapshot = mvccTracker.snapshot();
+ }
+
+ assert mvccSnapshot != null;
+ }
boolean loc = nodes.size() == 1 && F.first(nodes).id().equals(cctx.localNodeId());
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java
index e51b7d0..b2cbe05 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java
@@ -106,10 +106,6 @@ public abstract class CacheMvccAbstractBasicCoordinatorFailoverTest extends Cach
}
catch (ClusterTopologyException e) {
info("Expected exception: " + e);
-
- assertNotNull(e.retryReadyFuture());
-
- e.retryReadyFuture().get();
}
catch (CacheException e) {
info("Expected exception: " + e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractFeatureTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractFeatureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractFeatureTest.java
index f5172c8..3ff8846 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractFeatureTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractFeatureTest.java
@@ -69,15 +69,22 @@ public abstract class CacheMvccAbstractFeatureTest extends CacheMvccAbstractTest
@Override protected void beforeTestsStarted() throws Exception {
super.beforeTestsStarted();
+ cleanPersistenceDir();
+
startGrids(4);
node = grid(0);
}
/** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- super.beforeTest();
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ super.afterTestsStopped();
+ }
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
CacheConfiguration<Integer, Person> ccfg = new CacheConfiguration<>(CACHE_NAME);
ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
@@ -93,8 +100,6 @@ public abstract class CacheMvccAbstractFeatureTest extends CacheMvccAbstractTest
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
node.destroyCache(CACHE_NAME);
-
- super.afterTest();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java
index 1abb45f..6e22f44 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java
@@ -93,9 +93,11 @@ import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.cache.CacheMode.REPLICATED;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SCAN;
import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SQL;
import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SQL_SUM;
import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.WriteMode.DML;
+import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.WriteMode.PUT;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
@@ -868,6 +870,9 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
ReadMode readMode,
WriteMode writeMode
) throws Exception {
+ if(readMode == SCAN && writeMode == PUT)
+ fail("https://issues.apache.org/jira/browse/IGNITE-7764");
+
final int RANGE = 20;
final int writers = 4;
@@ -1052,6 +1057,9 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
)
throws Exception
{
+ if(readMode == SCAN && writeMode == PUT)
+ fail("https://issues.apache.org/jira/browse/IGNITE-7764");
+
final int TOTAL = 20;
assert N <= TOTAL;
@@ -1501,7 +1509,7 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
*
* @throws Exception If failed.
*/
- private void verifyOldVersionsCleaned() throws Exception {
+ protected void verifyOldVersionsCleaned() throws Exception {
runVacuumSync();
// Check versions.
@@ -1531,8 +1539,8 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
if (!cctx.userCache() || !cctx.group().mvccEnabled())
continue;
- for (Object e : cache.withKeepBinary()) {
- IgniteBiTuple entry = (IgniteBiTuple)e;
+ for (Iterator it = cache.withKeepBinary().iterator(); it.hasNext(); ) {
+ IgniteBiTuple entry = (IgniteBiTuple)it.next();
KeyCacheObject key = cctx.toCacheKeyObject(entry.getKey());
@@ -1541,9 +1549,12 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
if (vers.size() > 1) {
if (failIfNotCleaned)
- fail("[key=" + key.value(null, false) + "; vers=" + vers + ']');
- else
+ fail("[key=" + key.value(null, false) + "; vers=" + vers + ']');
+ else {
+ U.closeQuiet((AutoCloseable)it);
+
return false;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccClusterRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccClusterRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccClusterRestartTest.java
index f9ab7bc..76a8604 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccClusterRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccClusterRestartTest.java
@@ -74,25 +74,29 @@ public class CacheMvccClusterRestartTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void afterTestsStopped() throws Exception {
- cleanPersistenceDir();
-
super.afterTestsStopped();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
}
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
- super.beforeTest();
+ fail("https://issues.apache.org/jira/browse/IGNITE-9394");
cleanPersistenceDir();
+
+ super.beforeTest();
}
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
- cleanPersistenceDir();
+ super.afterTest();
stopAllGrids();
- super.afterTest();
+ cleanPersistenceDir();
}
/**
@@ -124,8 +128,6 @@ public class CacheMvccClusterRestartTest extends GridCommonAbstractTest {
private void restart1(int srvBefore, int srvAfter) throws Exception {
Ignite srv0 = startGridsMultiThreaded(srvBefore);
- srv0.active(true);
-
IgniteCache<Object, Object> cache = srv0.createCache(cacheConfiguration());
Set<Integer> keys = new HashSet<>(primaryKeys(cache, 1, 0));
@@ -141,8 +143,6 @@ public class CacheMvccClusterRestartTest extends GridCommonAbstractTest {
srv0 = startGridsMultiThreaded(srvAfter);
- srv0.active(true);
-
cache = srv0.cache(DEFAULT_CACHE_NAME);
Map<Object, Object> res = cache.getAll(keys);
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
index 01268de..f1519df 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
@@ -55,6 +55,7 @@ import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
@@ -416,6 +417,8 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @throws Exception If failed.
*/
public void testTxReadIsolationSimple() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-7764");
+
Ignite srv0 = startGrids(4);
client = true;
@@ -2263,8 +2266,6 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
checkActiveQueriesCleanup(ignite(0));
- verifyCoordinatorInternalState();
-
try {
fut.get();
}
@@ -2466,10 +2467,6 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
}
catch (ClusterTopologyException e) {
info("Expected exception: " + e);
-
- assertNotNull(e.retryReadyFuture());
-
- e.retryReadyFuture().get();
}
return null;
@@ -2817,7 +2814,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
IgniteCache cache = node.createCache(cacheConfiguration(PARTITIONED, FULL_SYNC, 2, 64));
- final int KEYS = 10_000;
+ final int KEYS = 1_000;
Map<Object, Object> data = new HashMap<>();
@@ -2905,6 +2902,8 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @throws Exception If failed.
*/
public void testUpdate_N_Objects_ClientServer_Backups1_Scan() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-7764");
+
int[] nValues = {3, 5, 10};
for (int n : nValues) {
@@ -3262,7 +3261,9 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
CacheDataRow row = cctx.offheap().read(cctx, key0);
- checkRow(cctx, row, key0, vers.get(0).get1());
+ Object val = ((CacheObject)vers.get(0).get1()).value(cctx.cacheObjectContext(), false);
+
+ checkRow(cctx, row, key0, val);
for (IgniteBiTuple<Object, MvccVersion> ver : vers) {
MvccVersion cntr = ver.get2();
@@ -3272,18 +3273,20 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
row = cctx.offheap().mvccRead(cctx, key0, readVer);
- checkRow(cctx, row, key0, ver.get1());
+ Object verVal = ((CacheObject)ver.get1()).value(cctx.cacheObjectContext(), false);
+
+ checkRow(cctx, row, key0, verVal);
}
checkRow(cctx,
cctx.offheap().mvccRead(cctx, key0, version(vers.get(0).get2().coordinatorVersion() + 1, 1)),
key0,
- vers.get(0).get1());
+ val);
checkRow(cctx,
cctx.offheap().mvccRead(cctx, key0, version(vers.get(0).get2().coordinatorVersion(), vers.get(0).get2().counter() + 1)),
key0,
- vers.get(0).get1());
+ val);
MvccSnapshotResponse ver = version(vers.get(0).get2().coordinatorVersion(), 100000);
@@ -3296,8 +3299,11 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
if (v == vers.size() - 1)
assertNull(row);
- else
- checkRow(cctx, row, key0, vers.get(v + 1).get1());
+ else {
+ Object nextVal = ((CacheObject)vers.get(v + 1).get1()).value(cctx.cacheObjectContext(), false);
+
+ checkRow(cctx, row, key0, nextVal);
+ }
}
}
@@ -3316,7 +3322,8 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @throws Exception If failed.
*/
public void testExpiration() throws Exception {
- fail("https://issues.apache.org/jira/browse/IGNITE-7956");
+ fail("https://issues.apache.org/jira/browse/IGNITE-7311");
+
final IgniteEx node = startGrid(0);
IgniteCache cache = node.createCache(cacheConfiguration(PARTITIONED, FULL_SYNC, 1, 64));
@@ -3370,6 +3377,8 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @throws Exception If failed.
*/
public void testChangeExpireTime() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-7311");
+
final IgniteEx node = startGrid(0);
IgniteCache cache = node.createCache(cacheConfiguration(PARTITIONED, FULL_SYNC, 1, 64));
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java
new file mode 100644
index 0000000..c1af42b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.datastreamer;
+
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+
+/**
+ * Check DataStreamer with Mvcc enabled.
+ */
+public class DataStreamProcessorMvccSelfTest extends DataStreamProcessorSelfTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration igniteConfiguration = super.getConfiguration(igniteInstanceName);
+
+ CacheConfiguration[] cacheConfigurations = igniteConfiguration.getCacheConfiguration();
+
+ assert cacheConfigurations == null || cacheConfigurations.length == 0
+ || (cacheConfigurations.length == 1 && cacheConfigurations[0].getAtomicityMode() == TRANSACTIONAL);
+
+ igniteConfiguration.setMvccEnabled(true);
+
+ return igniteConfiguration;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testPartitioned() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8149");
+
+ super.testPartitioned();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testColocated() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8149");
+
+ super.testColocated();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testReplicated() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8149");
+
+ super.testReplicated();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testUpdateStore() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8582");
+
+ super.testUpdateStore();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 55086f3..273456a 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -83,7 +83,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartit
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
import org.apache.ignite.internal.processors.cache.local.GridLocalCache;
@@ -91,10 +90,6 @@ import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabase
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
-import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
-import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
-import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask;
-import org.apache.ignite.internal.util.lang.GridAbsClosure;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.G;
@@ -102,13 +97,10 @@ import org.apache.ignite.internal.util.typedef.PA;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.LT;
import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiInClosure;
-import org.apache.ignite.lang.IgniteClosure;
import org.apache.ignite.internal.visor.VisorTaskArgument;
-import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask;
import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
-import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult;
import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskV2;
+import org.apache.ignite.lang.IgniteBiInClosure;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgnitePredicate;
@@ -128,7 +120,6 @@ import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.cache.CacheMode.LOCAL;
import static org.apache.ignite.cache.CacheRebalanceMode.NONE;
import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isNearEnabled;
-import static org.apache.ignite.internal.processors.cache.GridCacheUtils.retryTopologySafe;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java
index b5275b9..f87b14d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite.java
@@ -30,7 +30,7 @@ import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccScanQueryWithCo
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccSizeWithConcurrentTransactionTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccTransactionsTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccVacuumTest;
-import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorMvccSeflTest;
+import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessorMvccSelfTest;
/**
*
@@ -42,19 +42,25 @@ public class IgniteCacheMvccTestSuite extends TestSuite {
public static TestSuite suite() {
TestSuite suite = new TestSuite("IgniteCache MVCC Test Suite");
+ // Basic tests.
suite.addTestSuite(CacheMvccTransactionsTest.class);
suite.addTestSuite(CacheMvccProcessorTest.class);
- suite.addTestSuite(CacheMvccClusterRestartTest.class);
+ suite.addTestSuite(CacheMvccVacuumTest.class);
suite.addTestSuite(CacheMvccConfigurationValidationTest.class);
+
+ suite.addTestSuite(DataStreamProcessorMvccSelfTest.class);
suite.addTestSuite(CacheMvccOperationChecksTest.class);
+
+ // Concurrent ops tests.
suite.addTestSuite(CacheMvccIteratorWithConcurrentTransactionTest.class);
suite.addTestSuite(CacheMvccLocalEntriesWithConcurrentTransactionTest.class);
suite.addTestSuite(CacheMvccScanQueryWithConcurrentTransactionTest.class);
suite.addTestSuite(CacheMvccSizeWithConcurrentTransactionTest.class);
- suite.addTestSuite(CacheMvccVacuumTest.class);
+
+ // Failover tests.
+ suite.addTestSuite(CacheMvccClusterRestartTest.class);
suite.addTestSuite(CacheMvccPartitionedCoordinatorFailoverTest.class);
suite.addTestSuite(CacheMvccReplicatedCoordinatorFailoverTest.class);
- suite.addTestSuite(DataStreamProcessorMvccSeflTest.class);
return suite;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2FieldsIterator.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2FieldsIterator.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2FieldsIterator.java
index 05df754..e9f293c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2FieldsIterator.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2FieldsIterator.java
@@ -57,7 +57,7 @@ public class H2FieldsIterator extends H2ResultSetIterator<List<?>> {
}
/** {@inheritDoc} */
- @Override public void onClose() {
+ @Override public void onClose() throws IgniteCheckedException {
try {
super.onClose();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIterator.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIterator.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIterator.java
index e84ca04..814e83d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIterator.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIterator.java
@@ -26,7 +26,6 @@ import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2ValueCacheObject;
import org.apache.ignite.internal.util.GridCloseableIteratorAdapter;
import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
import org.h2.jdbc.JdbcResultSet;
import org.h2.result.ResultInterface;
import org.h2.value.Value;
@@ -99,13 +98,13 @@ public abstract class H2ResultSetIterator<T> extends GridCloseableIteratorAdapte
/**
* @return {@code true} If next row was fetched successfully.
*/
- private boolean fetchNext() {
+ private boolean fetchNext() throws IgniteCheckedException {
if (data == null)
return false;
try {
- if (!data.next()){
- onClose();
+ if (!data.next()) {
+ close();
return false;
}
@@ -138,7 +137,7 @@ public abstract class H2ResultSetIterator<T> extends GridCloseableIteratorAdapte
}
/** {@inheritDoc} */
- @Override public boolean onHasNext() {
+ @Override public boolean onHasNext() throws IgniteCheckedException {
return hasRow || (hasRow = fetchNext());
}
@@ -164,15 +163,21 @@ public abstract class H2ResultSetIterator<T> extends GridCloseableIteratorAdapte
}
/** {@inheritDoc} */
- @Override public void onClose(){
+ @Override public void onClose() throws IgniteCheckedException {
if (data == null)
// Nothing to close.
return;
- U.closeQuiet(data);
-
- res = null;
- data = null;
+ try {
+ data.close();
+ }
+ catch (SQLException e) {
+ throw new IgniteSQLException(e);
+ }
+ finally {
+ res = null;
+ data = null;
+ }
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsComandsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsComandsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsComandsSelfTest.java
new file mode 100644
index 0000000..8b3fbe3
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsComandsSelfTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.index;
+
+import java.util.concurrent.Callable;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class SqlTransactionsComandsSelfTest extends AbstractSchemaSelfTest {
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ startGrid(commonConfiguration(0));
+
+ super.execute(grid(0), "CREATE TABLE INTS(k int primary key, v int) WITH \"wrap_value=false,cache_name=ints," +
+ "atomicity=transactional\"");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ super.afterTestsStopped();
+ }
+
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testBeginWithMvccDisabledThrows() throws Exception {
+ checkMvccDisabledBehavior("BEGIN");
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testCommitWithMvccDisabledThrows() throws Exception {
+ checkMvccDisabledBehavior("COMMIT");
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRollbackWithMvccDisabledThrows() throws Exception {
+ checkMvccDisabledBehavior("rollback");
+ }
+
+ /**
+ * @param sql Operation to test.
+ * @throws Exception if failed.
+ */
+ private void checkMvccDisabledBehavior(String sql) throws Exception {
+ try (IgniteEx node = startGrid(commonConfiguration(1))) {
+ GridTestUtils.assertThrows(null, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ execute(node, sql);
+
+ return null;
+ }
+ }, IgniteSQLException.class, "MVCC must be enabled in order to invoke transactional operation: " + sql);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsCommandsWithMvccEnabledSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsCommandsWithMvccEnabledSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsCommandsWithMvccEnabledSelfTest.java
new file mode 100644
index 0000000..55dd59a
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsCommandsWithMvccEnabledSelfTest.java
@@ -0,0 +1,420 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.index;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy;
+import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionState;
+
+/**
+ * Tests to check behavior regarding transactions started via SQL.
+ */
+public class SqlTransactionsCommandsWithMvccEnabledSelfTest extends AbstractSchemaSelfTest {
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ startGrid(commonConfiguration(0).setMvccEnabled(true));
+
+ super.execute(node(), "CREATE TABLE INTS(k int primary key, v int) WITH \"wrap_value=false,cache_name=ints," +
+ "atomicity=transactional\"");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ super.afterTestsStopped();
+ }
+
+ /**
+ * Test that BEGIN opens a transaction.
+ */
+ public void testBegin() {
+ execute(node(), "BEGIN");
+
+ assertTxPresent();
+
+ assertTxState(tx(), TransactionState.ACTIVE);
+ }
+
+ /**
+ * Test that COMMIT commits a transaction.
+ */
+ public void testCommit() {
+ execute(node(), "BEGIN WORK");
+
+ assertTxPresent();
+
+ Transaction tx = tx();
+
+ assertTxState(tx, TransactionState.ACTIVE);
+
+ execute(node(), "COMMIT TRANSACTION");
+
+ assertTxState(tx, TransactionState.COMMITTED);
+
+ assertSqlTxNotPresent();
+ }
+
+ /**
+ * Test that COMMIT without a transaction yields nothing.
+ */
+ public void testCommitNoTransaction() {
+ execute(node(), "COMMIT");
+ }
+
+ /**
+ * Test that ROLLBACK without a transaction yields nothing.
+ */
+ public void testRollbackNoTransaction() {
+ execute(node(), "ROLLBACK");
+ }
+
+ /**
+ * Test that ROLLBACK rolls back a transaction.
+ */
+ public void testRollback() {
+ execute(node(), "BEGIN TRANSACTION");
+
+ assertTxPresent();
+
+ Transaction tx = tx();
+
+ assertTxState(tx, TransactionState.ACTIVE);
+
+ execute(node(), "ROLLBACK TRANSACTION");
+
+ assertTxState(tx, TransactionState.ROLLED_BACK);
+
+ assertSqlTxNotPresent();
+ }
+
+ /**
+ * Test that attempting to perform various SQL operations within non SQL transaction yields an exception.
+ */
+ public void testSqlOperationsWithinNonSqlTransaction() {
+ assertSqlOperationWithinNonSqlTransactionThrows("COMMIT");
+
+ assertSqlOperationWithinNonSqlTransactionThrows("ROLLBACK");
+
+ assertSqlOperationWithinNonSqlTransactionThrows("SELECT * from ints");
+
+ assertSqlOperationWithinNonSqlTransactionThrows("DELETE from ints");
+
+ assertSqlOperationWithinNonSqlTransactionThrows("INSERT INTO ints(k, v) values(10, 15)");
+
+ assertSqlOperationWithinNonSqlTransactionThrows("MERGE INTO ints(k, v) values(10, 15)");
+
+ assertSqlOperationWithinNonSqlTransactionThrows("UPDATE ints SET v = 100 WHERE k = 5");
+
+ assertSqlOperationWithinNonSqlTransactionThrows("create index idx on ints(v)");
+
+ assertSqlOperationWithinNonSqlTransactionThrows("CREATE TABLE T(k int primary key, v int)");
+ }
+
+ /**
+ * Check that trying to run given SQL statement both locally and in distributed mode yields an exception
+ * if transaction already has been marked as being of SQL type.
+ * @param sql SQL statement.
+ */
+ private void assertSqlOperationWithinNonSqlTransactionThrows(final String sql) {
+ try (Transaction ignored = node().transactions().txStart()) {
+ node().cache("ints").put(1, 1);
+
+ assertSqlException(new RunnableX() {
+ @Override public void run() throws Exception {
+ execute(node(), sql);
+ }
+ }, IgniteQueryErrorCode.TRANSACTION_TYPE_MISMATCH);
+ }
+
+ try (Transaction ignored = node().transactions().txStart()) {
+ node().cache("ints").put(1, 1);
+
+ assertSqlException(new RunnableX() {
+ @Override public void run() throws Exception {
+ node().cache("ints").query(new SqlFieldsQuery(sql).setLocal(true)).getAll();
+ }
+ }, IgniteQueryErrorCode.TRANSACTION_TYPE_MISMATCH);
+ }
+ }
+
+ /**
+ * Test that attempting to perform a cache API operation from within an SQL transaction fails.
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ private void checkCacheOperationThrows(final String opName, final Object... args) {
+ execute(node(), "BEGIN");
+
+ try {
+ GridTestUtils.assertThrows(null, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ try {
+ // We need to detect types based on arguments due to multiple overloads.
+ Class[] types;
+
+ if (F.isEmpty(args))
+ types = (Class[]) X.EMPTY_OBJECT_ARRAY;
+ else {
+ types = new Class[args.length];
+
+ for (int i = 0; i < args.length; i++)
+ types[i] = argTypeForObject(args[i]);
+ }
+
+ Object res = U.invoke(GatewayProtectedCacheProxy.class, node().cache("ints"),
+ opName, types, args);
+
+ if (opName.endsWith("Async"))
+ ((IgniteFuture)res).get();
+ }
+ catch (IgniteCheckedException e) {
+ if (e.getCause() != null) {
+ try {
+ if (e.getCause().getCause() != null)
+ throw (Exception)e.getCause().getCause();
+ else
+ fail();
+ }
+ catch (IgniteException e1) {
+ // Some public API methods don't have IgniteCheckedException on their signature
+ // and thus may wrap it into an IgniteException.
+ if (e1.getCause() != null)
+ throw (Exception)e1.getCause();
+ else
+ fail();
+ }
+ }
+ else
+ fail();
+ }
+
+ return null;
+ }
+ }, IgniteCheckedException.class,
+ "SQL queries and cache operations may not be used in the same transaction.");
+ }
+ finally {
+ try {
+ execute(node(), "ROLLBACK");
+ }
+ catch (Throwable e) {
+ // No-op.
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private static Class<?> argTypeForObject(Object arg) {
+ if (arg instanceof Set)
+ return Set.class;
+ else if (arg instanceof Map)
+ return Map.class;
+ else if (arg.getClass().getName().startsWith("java.lang."))
+ return Object.class;
+ else if (arg instanceof CacheEntryProcessor)
+ return CacheEntryProcessor.class;
+ else if (arg instanceof EntryProcessor)
+ return EntryProcessor.class;
+ else
+ return arg.getClass();
+ }
+
+ /**
+ * Test that attempting to perform a cache PUT operation from within an SQL transaction fails.
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testCacheOperationsFromSqlTransaction() {
+ checkCacheOperationThrows("get", 1);
+
+ checkCacheOperationThrows("getAsync", 1);
+
+ checkCacheOperationThrows("getEntry", 1);
+
+ checkCacheOperationThrows("getEntryAsync", 1);
+
+ checkCacheOperationThrows("getAndPut", 1, 1);
+
+ checkCacheOperationThrows("getAndPutAsync", 1, 1);
+
+ checkCacheOperationThrows("getAndPutIfAbsent", 1, 1);
+
+ checkCacheOperationThrows("getAndPutIfAbsentAsync", 1, 1);
+
+ checkCacheOperationThrows("getAndReplace", 1, 1);
+
+ checkCacheOperationThrows("getAndReplaceAsync", 1, 1);
+
+ checkCacheOperationThrows("getAndRemove", 1);
+
+ checkCacheOperationThrows("getAndRemoveAsync", 1);
+
+ checkCacheOperationThrows("containsKey", 1);
+
+ checkCacheOperationThrows("containsKeyAsync", 1);
+
+ checkCacheOperationThrows("put", 1, 1);
+
+ checkCacheOperationThrows("putAsync", 1, 1);
+
+ checkCacheOperationThrows("putIfAbsent", 1, 1);
+
+ checkCacheOperationThrows("putIfAbsentAsync", 1, 1);
+
+ checkCacheOperationThrows("remove", 1);
+
+ checkCacheOperationThrows("removeAsync", 1);
+
+ checkCacheOperationThrows("remove", 1, 1);
+
+ checkCacheOperationThrows("removeAsync", 1, 1);
+
+ checkCacheOperationThrows("replace", 1, 1);
+
+ checkCacheOperationThrows("replaceAsync", 1, 1);
+
+ checkCacheOperationThrows("replace", 1, 1, 1);
+
+ checkCacheOperationThrows("replaceAsync", 1, 1, 1);
+
+ checkCacheOperationThrows("getAll", new HashSet<>(Arrays.asList(1, 2)));
+
+ checkCacheOperationThrows("containsKeys", new HashSet<>(Arrays.asList(1, 2)));
+
+ checkCacheOperationThrows("getEntries", new HashSet<>(Arrays.asList(1, 2)));
+
+ checkCacheOperationThrows("putAll", Collections.singletonMap(1, 1));
+
+ checkCacheOperationThrows("removeAll", new HashSet<>(Arrays.asList(1, 2)));
+
+ checkCacheOperationThrows("getAllAsync", new HashSet<>(Arrays.asList(1, 2)));
+
+ checkCacheOperationThrows("containsKeysAsync", new HashSet<>(Arrays.asList(1, 2)));
+
+ checkCacheOperationThrows("getEntriesAsync", new HashSet<>(Arrays.asList(1, 2)));
+
+ checkCacheOperationThrows("putAllAsync", Collections.singletonMap(1, 1));
+
+ checkCacheOperationThrows("removeAllAsync", new HashSet<>(Arrays.asList(1, 2)));
+
+ checkCacheOperationThrows("invoke", 1, ENTRY_PROC, X.EMPTY_OBJECT_ARRAY);
+
+ checkCacheOperationThrows("invoke", 1, CACHE_ENTRY_PROC, X.EMPTY_OBJECT_ARRAY);
+
+ checkCacheOperationThrows("invokeAsync", 1, ENTRY_PROC, X.EMPTY_OBJECT_ARRAY);
+
+ checkCacheOperationThrows("invokeAsync", 1, CACHE_ENTRY_PROC, X.EMPTY_OBJECT_ARRAY);
+
+ checkCacheOperationThrows("invokeAll", Collections.singletonMap(1, CACHE_ENTRY_PROC), X.EMPTY_OBJECT_ARRAY);
+
+ checkCacheOperationThrows("invokeAll", Collections.singleton(1), CACHE_ENTRY_PROC, X.EMPTY_OBJECT_ARRAY);
+
+ checkCacheOperationThrows("invokeAll", Collections.singleton(1), ENTRY_PROC, X.EMPTY_OBJECT_ARRAY);
+
+ checkCacheOperationThrows("invokeAllAsync", Collections.singletonMap(1, CACHE_ENTRY_PROC),
+ X.EMPTY_OBJECT_ARRAY);
+
+ checkCacheOperationThrows("invokeAllAsync", Collections.singleton(1), CACHE_ENTRY_PROC, X.EMPTY_OBJECT_ARRAY);
+
+ checkCacheOperationThrows("invokeAllAsync", Collections.singleton(1), ENTRY_PROC, X.EMPTY_OBJECT_ARRAY);
+ }
+
+ /** */
+ private final static EntryProcessor<Integer, Integer, Object> ENTRY_PROC =
+ new EntryProcessor<Integer, Integer, Object>() {
+ @Override public Object process(MutableEntry<Integer, Integer> entry, Object... arguments)
+ throws EntryProcessorException {
+ return null;
+ }
+ };
+
+ /** */
+ private final static CacheEntryProcessor<Integer, Integer, Object> CACHE_ENTRY_PROC =
+ new CacheEntryProcessor<Integer, Integer, Object>() {
+ @Override public Object process(MutableEntry<Integer, Integer> entry, Object... arguments)
+ throws EntryProcessorException {
+ return null;
+ }
+ };
+
+ /**
+ * @return Node.
+ */
+ private IgniteEx node() {
+ return grid(0);
+ }
+
+ /**
+ * @return Currently open transaction.
+ */
+ private Transaction tx() {
+ return node().transactions().tx();
+ }
+
+ /**
+ * Check that there's an open transaction with SQL flag.
+ */
+ private void assertTxPresent() {
+ assertNotNull(tx());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected List<List<?>> execute(Ignite node, String sql) {
+ return node.cache("ints").query(new SqlFieldsQuery(sql).setSchema(QueryUtils.DFLT_SCHEMA)).getAll();
+ }
+
+ /**
+ * Check that there's no open transaction.
+ */
+ private void assertSqlTxNotPresent() {
+ assertNull(tx());
+ }
+
+ /**
+ * Check transaction state.
+ */
+ private static void assertTxState(Transaction tx, TransactionState state) {
+ assertEquals(state, tx.state());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBackupsAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBackupsAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBackupsAbstractTest.java
index 71e004b..998cb76 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBackupsAbstractTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccBackupsAbstractTest.java
@@ -71,7 +71,7 @@ public abstract class CacheMvccBackupsAbstractTest extends CacheMvccAbstractTest
public void testBackupsCoherenceSimple() throws Exception {
disableScheduledVacuum = true;
- ccfg = cacheConfiguration(cacheMode(), FULL_SYNC, 2, DFLT_PARTITION_COUNT)
+ ccfg = cacheConfiguration(cacheMode(), FULL_SYNC, 2, 10)
.setIndexedTypes(Integer.class, Integer.class);
final int KEYS_CNT = 5_000;
@@ -184,10 +184,10 @@ public abstract class CacheMvccBackupsAbstractTest extends CacheMvccAbstractTest
public void testBackupsCoherenceWithLargeOperations() throws Exception {
disableScheduledVacuum = true;
- ccfg = cacheConfiguration(cacheMode(), FULL_SYNC, 1, DFLT_PARTITION_COUNT)
+ ccfg = cacheConfiguration(cacheMode(), FULL_SYNC, 1, 10)
.setIndexedTypes(Integer.class, Integer.class);
- final int KEYS_CNT = 50_000;
+ final int KEYS_CNT = 5_000;
assert KEYS_CNT % 2 == 0;
startGrids(2);
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlQueriesAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlQueriesAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlQueriesAbstractTest.java
index 4ad667b9..9e0b02f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlQueriesAbstractTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlQueriesAbstractTest.java
@@ -629,6 +629,8 @@ public abstract class CacheMvccSqlQueriesAbstractTest extends CacheMvccAbstractT
* @throws Exception If failed.
*/
public void testDistributedJoinSimple() throws Exception {
+ disableScheduledVacuum = true; //TODO: IGNITE-9446: remove this after races in vacuum will be fixed.
+
startGridsMultiThreaded(4);
Ignite srv0 = ignite(0);
http://git-wip-us.apache.org/repos/asf/ignite/blob/254a6529/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesWithReducerAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesWithReducerAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesWithReducerAbstractTest.java
index ade3763..69cf108 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesWithReducerAbstractTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesWithReducerAbstractTest.java
@@ -634,7 +634,7 @@ public abstract class CacheMvccSqlTxQueriesWithReducerAbstractTest extends Cache
barrier.await();
String sqlText = "UPDATE MvccTestSqlIndexValue t SET idxVal1=" +
- "(SELECT _val FROM \"int\".Integer WHERE t._key = _key ORDER BY _key)";
+ "(SELECT _val FROM \"int\".Integer WHERE _key >= 5 AND _key <= 5 ORDER BY _key) WHERE _key = 5";
qry = new SqlFieldsQuery(sqlText);
[34/50] [abbrv] ignite git commit: Revert TOPIC_MONGO removal in
"IGNITE-9361 Remove IgniteInternalCache.isMongo*Cache() and other remnants."
Posted by ag...@apache.org.
Revert TOPIC_MONGO removal in "IGNITE-9361 Remove IgniteInternalCache.isMongo*Cache() and other remnants."
This partially reverts commit de30a86e740067ed239602fb5dc28396144fd446. - Fixes #4687.
Signed-off-by: Dmitriy Govorukhin <dm...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ed6bf5ac
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ed6bf5ac
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ed6bf5ac
Branch: refs/heads/ignite-5960
Commit: ed6bf5ac2af7886d06d1cbab5738e48c89269379
Parents: 1894541
Author: Ilya Kasnacheev <il...@gmail.com>
Authored: Wed Sep 5 17:53:28 2018 +0300
Committer: Dmitriy Govorukhin <dm...@gmail.com>
Committed: Wed Sep 5 17:53:28 2018 +0300
----------------------------------------------------------------------
.../src/main/java/org/apache/ignite/internal/GridTopic.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed6bf5ac/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index 5610b2a..98a4d8d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -81,6 +81,12 @@ public enum GridTopic {
/** */
TOPIC_CONTINUOUS,
+ /**
+ * @deprecated Should be removed in Apache Ignite 3.0.
+ */
+ @Deprecated
+ TOPIC_MONGO,
+
/** */
TOPIC_TIME_SYNC,
[50/50] [abbrv] ignite git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/ignite into ignite-5960
Posted by ag...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/ignite into ignite-5960
Conflicts:
modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a7ee714e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a7ee714e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a7ee714e
Branch: refs/heads/ignite-5960
Commit: a7ee714e6dd04d5380fb8bc20003ef721bf5ec92
Parents: 8531961 00e935d
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Fri Sep 7 16:48:37 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Sep 7 16:48:37 2018 +0300
----------------------------------------------------------------------
.gitignore | 3 +
bin/control.bat | 30 +-
bin/ignite.bat | 30 +-
bin/ignite.sh | 16 +-
bin/ignitevisorcmd.bat | 32 +-
bin/ignitevisorcmd.sh | 8 +-
bin/include/functions.sh | 44 +-
examples/DEVNOTES.txt | 11 +
.../examples/ml/MLExamplesCommonArgs.java | 31 -
.../clustering/KMeansClusterizationExample.java | 47 +-
.../AlgorithmSpecificDatasetExample.java | 35 +-
.../ml/dataset/CacheBasedDatasetExample.java | 29 +-
.../ml/dataset/LocalDatasetExample.java | 84 -
.../ignite/examples/ml/genetic/change/Coin.java | 62 +-
.../OptimizeMakeChangeFitnessFunction.java | 66 +-
.../change/OptimizeMakeChangeGAExample.java | 167 +-
.../OptimizeMakeChangeTerminateCriteria.java | 66 +-
.../helloworld/HelloWorldFitnessFunction.java | 39 +-
.../genetic/helloworld/HelloWorldGAExample.java | 109 +-
.../helloworld/HelloWorldTerminateCriteria.java | 73 +-
.../examples/ml/genetic/knapsack/Item.java | 43 +-
.../knapsack/KnapsackFitnessFunction.java | 43 +-
.../ml/genetic/knapsack/KnapsackGAExample.java | 165 +-
.../knapsack/KnapsackTerminateCriteria.java | 83 +-
.../ignite/examples/ml/genetic/movie/Movie.java | 51 +-
.../ml/genetic/movie/MovieFitnessFunction.java | 70 +-
.../ml/genetic/movie/MovieGAExample.java | 141 +-
.../genetic/movie/MovieTerminateCriteria.java | 71 +-
.../ml/knn/ANNClassificationExample.java | 317 ++
.../ml/knn/KNNClassificationExample.java | 50 +-
.../examples/ml/knn/KNNRegressionExample.java | 56 +-
.../examples/ml/nn/MLPTrainerExample.java | 17 +-
.../ml/preprocessing/BinarizationExample.java | 31 +-
.../ml/preprocessing/ImputingExample.java | 34 +-
.../ImputingWithMostFrequentValuesExample.java | 38 +-
.../ml/preprocessing/MinMaxScalerExample.java | 35 +-
.../ml/preprocessing/NormalizationExample.java | 31 +-
.../LinearRegressionLSQRTrainerExample.java | 41 +-
...ssionLSQRTrainerWithMinMaxScalerExample.java | 48 +-
.../LinearRegressionSGDTrainerExample.java | 43 +-
.../LogisticRegressionSGDTrainerExample.java | 45 +-
...gressionMultiClassClassificationExample.java | 46 +-
.../ml/selection/cv/CrossValidationExample.java | 10 +-
.../split/TrainTestDatasetSplitterExample.java | 43 +-
.../binary/SVMBinaryClassificationExample.java | 43 +-
.../SVMMultiClassClassificationExample.java | 45 +-
...ecisionTreeClassificationTrainerExample.java | 17 +-
.../DecisionTreeRegressionTrainerExample.java | 12 +-
.../GDBOnTreesClassificationTrainerExample.java | 21 +-
.../GDBOnTreesRegressionTrainerExample.java | 19 +-
.../RandomForestClassificationExample.java | 438 +--
.../RandomForestRegressionExample.java | 1091 +++---
.../ml/tutorial/Step_1_Read_and_Learn.java | 22 +-
.../examples/ml/tutorial/Step_2_Imputing.java | 21 +-
.../examples/ml/tutorial/Step_3_Categorial.java | 24 +-
.../Step_3_Categorial_with_One_Hot_Encoder.java | 26 +-
.../ml/tutorial/Step_4_Add_age_fare.java | 22 +-
.../examples/ml/tutorial/Step_5_Scaling.java | 40 +-
.../ignite/examples/ml/tutorial/Step_6_KNN.java | 48 +-
.../ml/tutorial/Step_7_Split_train_test.java | 43 +-
.../ignite/examples/ml/tutorial/Step_8_CV.java | 57 +-
.../ml/tutorial/Step_8_CV_with_Param_Grid.java | 53 +-
.../ml/tutorial/Step_9_Go_to_LogReg.java | 44 +-
.../examples/ml/tutorial/TitanicUtils.java | 2 +-
.../ml/tutorial/TutorialStepByStepExample.java | 4 +-
.../ignite/examples/ml/util/DatasetHelper.java | 57 +
.../examples/ml/util/MLExamplesCommonArgs.java | 31 +
.../ignite/examples/ml/util/TestCache.java | 77 +
.../ignite/examples/ml/util/package-info.java | 22 +
.../testsuites/IgniteExamplesMLTestSuite.java | 2 +-
idea/ignite_inspections.xml | 772 ++++
.../benchmarks/jmh/tree/BPlusTreeBenchmark.java | 2 +-
modules/camel/pom.xml | 1 -
.../camel/IgniteCamelStreamerTestSuite.java | 2 +-
modules/cassandra/store/pom.xml | 1 -
.../internal/jdbc2/JdbcStreamingSelfTest.java | 3 +-
.../rest/AbstractRestProcessorSelfTest.java | 2 +
.../JettyRestProcessorAbstractSelfTest.java | 2 +-
.../rest/JettyRestProcessorCommonSelfTest.java | 2 +
.../suite/IgniteJdbcDriverMvccTestSuite.java | 47 +
.../jdbc/suite/IgniteJdbcDriverTestSuite.java | 17 +-
.../jdbc/thin/JdbcThinAbstractSelfTest.java | 4 +-
...bcThinComplexDmlDdlCustomSchemaSelfTest.java | 78 +
.../thin/JdbcThinComplexDmlDdlSelfTest.java | 20 +-
.../JdbcThinConnectionMvccEnabledSelfTest.java | 378 ++
.../jdbc/thin/JdbcThinConnectionSelfTest.java | 188 +-
.../jdbc/thin/JdbcThinMetadataSelfTest.java | 49 +-
.../thin/JdbcThinStreamingAbstractSelfTest.java | 19 +-
.../JdbcThinStreamingNotOrderedSelfTest.java | 2 +-
.../thin/JdbcThinStreamingOrderedSelfTest.java | 2 +-
...ThinTransactionsAbstractComplexSelfTest.java | 1058 +++++
...sactionsClientAutoCommitComplexSelfTest.java | 33 +
...ctionsClientNoAutoCommitComplexSelfTest.java | 33 +
.../jdbc/thin/JdbcThinTransactionsSelfTest.java | 447 +++
...sactionsServerAutoCommitComplexSelfTest.java | 33 +
...ctionsServerNoAutoCommitComplexSelfTest.java | 33 +
...ThinTransactionsWithMvccEnabledSelfTest.java | 447 +++
modules/clients/src/test/keystore/ca/node01.jks | Bin 3719 -> 3719 bytes
modules/clients/src/test/keystore/ca/node02.jks | Bin 4598 -> 4599 bytes
modules/clients/src/test/keystore/ca/node03.jks | Bin 3754 -> 3753 bytes
.../clients/src/test/keystore/ca/oneindex.txt | 2 +-
.../clients/src/test/keystore/ca/twoindex.txt | 4 +-
.../TcpDiscoveryCloudIpFinderSelfTest.java | 2 +
.../ignite/codegen/MessageCodeGenerator.java | 4 +-
modules/core/pom.xml | 2 +-
.../java/org/apache/ignite/IgniteCompute.java | 14 +
.../apache/ignite/IgniteSystemProperties.java | 17 +
.../main/java/org/apache/ignite/Ignition.java | 8 +-
.../org/apache/ignite/cache/QueryEntity.java | 91 +-
.../org/apache/ignite/cluster/ClusterNode.java | 12 +-
.../configuration/CacheConfiguration.java | 1 +
.../configuration/DataStorageConfiguration.java | 42 +
.../DefaultCommunicationFailureResolver.java | 10 +-
.../configuration/IgniteConfiguration.java | 117 +-
.../org/apache/ignite/events/EventType.java | 15 +-
.../ignite/events/WalSegmentArchivedEvent.java | 13 +-
.../ignite/events/WalSegmentCompactedEvent.java | 46 +
.../apache/ignite/internal/GridComponent.java | 5 +-
.../ignite/internal/GridKernalContext.java | 6 +
.../ignite/internal/GridKernalContextImpl.java | 14 +-
.../org/apache/ignite/internal/GridTopic.java | 10 +-
.../ignite/internal/IgniteComputeImpl.java | 15 +
.../apache/ignite/internal/IgniteKernal.java | 34 +-
.../ignite/internal/IgniteNodeAttributes.java | 6 +-
.../org/apache/ignite/internal/IgnitionEx.java | 8 +-
.../ignite/internal/NodeStoppingException.java | 2 +-
.../ignite/internal/TransactionsMXBeanImpl.java | 20 +-
.../UnregisteredBinaryTypeException.java | 96 +
.../binary/BinaryCachingMetadataHandler.java | 2 +-
.../internal/binary/BinaryClassDescriptor.java | 9 +-
.../ignite/internal/binary/BinaryContext.java | 11 +-
.../internal/binary/BinaryFieldAccessor.java | 4 +
.../internal/binary/BinaryMetadataHandler.java | 3 +-
.../binary/BinaryNoopMetadataHandler.java | 2 +-
.../internal/binary/BinaryReaderExImpl.java | 2 +-
.../internal/binary/BinaryWriterExImpl.java | 13 +
.../binary/builder/BinaryBuilderSerializer.java | 2 +-
.../binary/builder/BinaryObjectBuilderImpl.java | 9 +-
.../internal/client/thin/ClientBinary.java | 2 +-
.../internal/client/thin/ClientChannel.java | 5 +
.../internal/client/thin/ClientUtils.java | 62 +-
.../internal/client/thin/ReliableChannel.java | 7 +
.../internal/client/thin/TcpClientCache.java | 3 +-
.../internal/client/thin/TcpClientChannel.java | 16 +-
.../internal/client/thin/TcpIgniteClient.java | 23 +-
.../internal/commandline/CommandHandler.java | 24 +-
.../jdbc/thin/ConnectionProperties.java | 10 +
.../jdbc/thin/ConnectionPropertiesImpl.java | 37 +-
.../internal/jdbc/thin/JdbcThinConnection.java | 33 +-
.../jdbc/thin/JdbcThinDatabaseMetadata.java | 11 +-
.../internal/jdbc/thin/JdbcThinStatement.java | 7 +-
.../internal/jdbc/thin/JdbcThinTcpIo.java | 32 +-
.../managers/communication/GridIoManager.java | 16 +
.../communication/GridIoMessageFactory.java | 139 +
.../internal/managers/discovery/DiscoCache.java | 17 +-
.../discovery/GridDiscoveryManager.java | 37 +-
.../managers/discovery/IgniteClusterNode.java | 7 -
.../pagemem/impl/PageMemoryNoStoreImpl.java | 8 +-
.../pagemem/store/IgnitePageStoreManager.java | 13 +
.../pagemem/wal/IgniteWriteAheadLogManager.java | 19 +-
.../internal/pagemem/wal/StorageException.java | 53 -
.../internal/pagemem/wal/record/WALRecord.java | 11 +-
.../delta/DataPageMvccMarkUpdatedRecord.java | 103 +
.../DataPageMvccUpdateNewTxStateHintRecord.java | 79 +
.../DataPageMvccUpdateTxStateHintRecord.java | 79 +
.../processors/affinity/AffinityAssignment.java | 15 +-
.../affinity/GridAffinityAssignment.java | 30 +-
.../affinity/GridAffinityAssignmentCache.java | 21 +-
.../affinity/GridAffinityProcessor.java | 2 +-
.../processors/affinity/GridAffinityUtils.java | 2 +-
.../affinity/HistoryAffinityAssignment.java | 11 +-
.../cache/CacheAffinitySharedManager.java | 69 +-
.../cache/CacheEntryInfoCollection.java | 12 +
.../processors/cache/CacheGroupContext.java | 34 +
.../cache/CacheGroupMetricsMXBeanImpl.java | 12 +-
.../cache/CacheInvokeDirectResult.java | 37 +
.../processors/cache/CacheInvokeResult.java | 6 +
.../processors/cache/CacheOperationContext.java | 2 +-
.../processors/cache/ClusterCachesInfo.java | 17 +-
.../processors/cache/ExchangeContext.java | 43 +-
.../cache/ExchangeDiscoveryEvents.java | 9 +-
.../cache/GatewayProtectedCacheProxy.java | 3 +
.../processors/cache/GridCacheAdapter.java | 113 +-
.../cache/GridCacheAffinityManager.java | 5 +
.../cache/GridCacheConcurrentMapImpl.java | 4 +-
.../processors/cache/GridCacheContext.java | 7 +
.../processors/cache/GridCacheEntryEx.java | 142 +-
.../processors/cache/GridCacheEntryInfo.java | 6 +-
.../cache/GridCacheEvictionManager.java | 19 +
.../processors/cache/GridCacheIoManager.java | 3 +-
.../processors/cache/GridCacheMapEntry.java | 1215 +++++-
.../cache/GridCacheMvccEntryInfo.java | 257 ++
.../processors/cache/GridCacheMvccManager.java | 2 +-
.../processors/cache/GridCacheOperation.java | 2 +-
.../GridCachePartitionExchangeManager.java | 118 +-
.../processors/cache/GridCacheProcessor.java | 121 +-
.../processors/cache/GridCacheProxyImpl.java | 24 -
.../processors/cache/GridCacheReturn.java | 28 +-
.../cache/GridCacheSharedContext.java | 11 +-
.../processors/cache/GridCacheTtlManager.java | 2 +-
.../cache/GridCacheUpdateTxResult.java | 98 +-
.../processors/cache/GridCacheUtils.java | 167 +-
.../cache/IgniteCacheOffheapManager.java | 489 ++-
.../cache/IgniteCacheOffheapManagerImpl.java | 1726 +++++++-
.../processors/cache/IgniteCacheProxyImpl.java | 10 +-
.../processors/cache/IgniteInternalCache.java | 14 -
.../binary/CacheObjectBinaryProcessor.java | 3 +-
.../binary/CacheObjectBinaryProcessorImpl.java | 26 +-
.../distributed/GridCacheTxFinishSync.java | 2 +-
.../distributed/GridCacheTxRecoveryFuture.java | 11 +
.../distributed/GridDistributedTxMapping.java | 17 +
.../GridDistributedTxPrepareRequest.java | 22 +
.../GridDistributedTxRemoteAdapter.java | 115 +-
.../distributed/dht/CompoundLockFuture.java | 112 +
.../cache/distributed/dht/DhtLockFuture.java | 30 +
.../cache/distributed/dht/ExceptionAware.java | 30 +
.../dht/GridClientPartitionTopology.java | 7 +
.../distributed/dht/GridDhtCacheAdapter.java | 68 +-
.../distributed/dht/GridDhtCacheEntry.java | 10 +-
.../cache/distributed/dht/GridDhtGetFuture.java | 17 +-
.../distributed/dht/GridDhtGetSingleFuture.java | 17 +-
.../distributed/dht/GridDhtLocalPartition.java | 58 +-
.../distributed/dht/GridDhtLockFuture.java | 15 +-
.../dht/GridDhtPartitionTopology.java | 4 +
.../dht/GridDhtPartitionTopologyImpl.java | 50 +-
.../dht/GridDhtPartitionsStateValidator.java | 11 +-
.../dht/GridDhtTransactionalCacheAdapter.java | 485 ++-
.../dht/GridDhtTxAbstractEnlistFuture.java | 1139 ++++++
.../distributed/dht/GridDhtTxFinishFuture.java | 116 +-
.../distributed/dht/GridDhtTxFinishRequest.java | 67 +-
.../cache/distributed/dht/GridDhtTxLocal.java | 11 +-
.../distributed/dht/GridDhtTxLocalAdapter.java | 96 +-
.../distributed/dht/GridDhtTxPrepareFuture.java | 120 +-
.../dht/GridDhtTxPrepareRequest.java | 31 +-
.../dht/GridDhtTxQueryEnlistFuture.java | 140 +
.../dht/GridDhtTxQueryEnlistRequest.java | 404 ++
.../dht/GridDhtTxQueryEnlistResponse.java | 205 +
.../dht/GridDhtTxQueryFirstEnlistRequest.java | 370 ++
.../dht/GridDhtTxQueryResultsEnlistFuture.java | 130 +
.../cache/distributed/dht/GridDhtTxRemote.java | 132 +
.../dht/GridPartitionedGetFuture.java | 92 +-
.../dht/GridPartitionedSingleGetFuture.java | 24 +-
.../dht/NearTxQueryEnlistResultHandler.java | 169 +
.../dht/PartitionUpdateCounters.java | 123 +
.../distributed/dht/PartitionsEvictManager.java | 5 +-
.../dht/atomic/DhtAtomicUpdateResult.java | 33 +-
.../dht/atomic/GridDhtAtomicCache.java | 176 +-
.../dht/colocated/GridDhtColocatedCache.java | 32 +-
.../colocated/GridDhtColocatedLockFuture.java | 499 ++-
.../preloader/CacheGroupAffinityMessage.java | 17 +-
.../dht/preloader/GridDhtForceKeysFuture.java | 43 +-
.../dht/preloader/GridDhtPartitionDemander.java | 52 +-
.../dht/preloader/GridDhtPartitionSupplier.java | 111 +-
.../GridDhtPartitionSupplyMessage.java | 6 +-
.../GridDhtPartitionSupplyMessageV2.java | 153 +
.../GridDhtPartitionsExchangeFuture.java | 292 +-
.../preloader/GridDhtPartitionsFullMessage.java | 4 +-
.../GridDhtPartitionsSingleMessage.java | 44 +-
.../dht/preloader/GridDhtPreloader.java | 43 +-
.../preloader/latch/ExchangeLatchManager.java | 2 +-
.../near/AckCoordinatorOnRollback.java | 54 +
.../distributed/near/GridNearAtomicCache.java | 2 +-
.../distributed/near/GridNearCacheAdapter.java | 10 -
.../distributed/near/GridNearGetFuture.java | 24 +-
.../distributed/near/GridNearGetRequest.java | 33 +-
.../distributed/near/GridNearLockFuture.java | 80 +-
...arOptimisticSerializableTxPrepareFuture.java | 94 +-
.../near/GridNearOptimisticTxPrepareFuture.java | 89 +-
...ridNearOptimisticTxPrepareFutureAdapter.java | 178 +-
.../GridNearPessimisticTxPrepareFuture.java | 195 +-
.../near/GridNearSingleGetRequest.java | 33 +-
.../near/GridNearTransactionalCache.java | 6 +-
.../near/GridNearTxAbstractEnlistFuture.java | 499 +++
.../near/GridNearTxFastFinishFuture.java | 13 +-
.../near/GridNearTxFinishAndAckFuture.java | 144 +
.../near/GridNearTxFinishFuture.java | 78 +-
.../near/GridNearTxFinishRequest.java | 30 +-
.../cache/distributed/near/GridNearTxLocal.java | 417 +-
.../near/GridNearTxPrepareFutureAdapter.java | 12 +-
.../near/GridNearTxPrepareRequest.java | 17 +
.../near/GridNearTxPrepareResponse.java | 34 +-
.../near/GridNearTxQueryEnlistFuture.java | 382 ++
.../near/GridNearTxQueryEnlistRequest.java | 616 +++
.../near/GridNearTxQueryEnlistResponse.java | 298 ++
.../GridNearTxQueryResultsEnlistFuture.java | 670 ++++
.../GridNearTxQueryResultsEnlistRequest.java | 578 +++
.../GridNearTxQueryResultsEnlistResponse.java | 202 +
.../distributed/near/GridNearTxRemote.java | 9 +-
.../near/GridNearTxSelectForUpdateFuture.java | 462 +++
.../distributed/near/NearTxFinishFuture.java | 21 +-
.../near/TxTopologyVersionFuture.java | 155 +
.../processors/cache/dr/GridCacheDrManager.java | 31 +
.../cache/dr/GridOsCacheDrManager.java | 13 +
.../processors/cache/local/GridLocalCache.java | 2 +-
.../local/atomic/GridLocalAtomicCache.java | 29 +-
.../processors/cache/mvcc/MvccCoordinator.java | 104 +
.../cache/mvcc/MvccDiscoveryData.java | 52 +
.../cache/mvcc/MvccEmptyLongList.java | 53 +
.../processors/cache/mvcc/MvccFuture.java | 66 +
.../processors/cache/mvcc/MvccLongList.java | 29 +
.../mvcc/MvccPreviousCoordinatorQueries.java | 222 ++
.../processors/cache/mvcc/MvccProcessor.java | 250 ++
.../cache/mvcc/MvccProcessorImpl.java | 2200 +++++++++++
.../processors/cache/mvcc/MvccQueryTracker.java | 102 +
.../cache/mvcc/MvccQueryTrackerImpl.java | 348 ++
.../processors/cache/mvcc/MvccSnapshot.java | 48 +
.../cache/mvcc/MvccSnapshotFuture.java | 43 +
.../mvcc/MvccSnapshotResponseListener.java | 35 +
.../cache/mvcc/MvccSnapshotWithoutTxs.java | 204 +
.../cache/mvcc/MvccUpdateVersionAware.java | 76 +
.../processors/cache/mvcc/MvccUtils.java | 879 +++++
.../processors/cache/mvcc/MvccVersion.java | 48 +
.../processors/cache/mvcc/MvccVersionAware.java | 76 +
.../processors/cache/mvcc/MvccVersionImpl.java | 193 +
.../cache/mvcc/NoOpMvccProcessor.java | 215 +
.../cache/mvcc/StaticMvccQueryTracker.java | 96 +
.../processors/cache/mvcc/VacuumMetrics.java | 102 +
.../cache/mvcc/VacuumMetricsReducer.java | 49 +
.../processors/cache/mvcc/VacuumTask.java | 63 +
.../cache/mvcc/msg/MvccAckRequestQueryCntr.java | 130 +
.../cache/mvcc/msg/MvccAckRequestQueryId.java | 130 +
.../cache/mvcc/msg/MvccAckRequestTx.java | 210 +
.../mvcc/msg/MvccAckRequestTxAndQueryCntr.java | 123 +
.../mvcc/msg/MvccAckRequestTxAndQueryId.java | 122 +
.../mvcc/msg/MvccActiveQueriesMessage.java | 132 +
.../cache/mvcc/msg/MvccFutureResponse.java | 130 +
.../processors/cache/mvcc/msg/MvccMessage.java | 35 +
.../mvcc/msg/MvccQuerySnapshotRequest.java | 130 +
.../cache/mvcc/msg/MvccSnapshotResponse.java | 320 ++
.../cache/mvcc/msg/MvccTxSnapshotRequest.java | 128 +
.../cache/mvcc/msg/MvccWaitTxsRequest.java | 159 +
.../processors/cache/mvcc/txlog/TxKey.java | 76 +
.../processors/cache/mvcc/txlog/TxLog.java | 584 +++
.../processors/cache/mvcc/txlog/TxLogIO.java | 73 +
.../cache/mvcc/txlog/TxLogInnerIO.java | 105 +
.../cache/mvcc/txlog/TxLogLeafIO.java | 105 +
.../processors/cache/mvcc/txlog/TxLogTree.java | 62 +
.../processors/cache/mvcc/txlog/TxRow.java | 44 +
.../processors/cache/mvcc/txlog/TxState.java | 37 +
.../cache/persistence/AllocatedPageTracker.java | 5 +-
.../cache/persistence/CacheDataRow.java | 3 +-
.../cache/persistence/CacheDataRowAdapter.java | 114 +-
.../cache/persistence/CacheSearchRow.java | 3 +-
.../persistence/DataRegionMetricsImpl.java | 16 +-
.../persistence/DatabaseLifecycleListener.java | 54 +
.../GridCacheDatabaseSharedManager.java | 118 +-
.../persistence/GridCacheOffheapManager.java | 328 +-
.../IgniteCacheDatabaseSharedManager.java | 66 +-
.../cache/persistence/IndexStorageImpl.java | 2 +-
.../processors/cache/persistence/RowStore.java | 28 +-
.../processors/cache/persistence/Storable.java | 14 +
.../cache/persistence/StorageException.java | 53 +
.../checkpoint/CheckpointHistory.java | 102 +-
.../cache/persistence/file/FilePageStore.java | 79 +-
.../persistence/file/FilePageStoreManager.java | 56 +-
.../file/PersistentStorageIOException.java | 47 -
.../persistence/freelist/AbstractFreeList.java | 20 +-
.../cache/persistence/freelist/FreeList.java | 12 +
.../persistence/metastorage/MetaStorage.java | 2 +-
.../metastorage/MetastorageDataRow.java | 11 +
.../metastorage/MetastorageTree.java | 2 +-
.../cache/persistence/pagemem/PageMemoryEx.java | 2 +-
.../persistence/pagemem/PageMemoryImpl.java | 2 +-
.../cache/persistence/tree/BPlusTree.java | 1312 +++++--
.../tree/CorruptedTreeException.java | 38 +
.../persistence/tree/io/AbstractDataPageIO.java | 35 +-
.../cache/persistence/tree/io/DataPageIO.java | 484 ++-
.../cache/persistence/tree/io/IOVersions.java | 7 +
.../cache/persistence/tree/io/PageIO.java | 133 +-
.../persistence/tree/io/SimpleDataPageIO.java | 6 -
.../tree/util/PageHandlerWrapper.java | 36 +
.../wal/AbstractWalRecordsIterator.java | 162 +-
.../cache/persistence/wal/FileDescriptor.java | 137 +
.../wal/FileWriteAheadLogManager.java | 325 +-
.../wal/FsyncModeFileWriteAheadLogManager.java | 267 +-
.../SingleSegmentLogicalRecordsIterator.java | 4 +-
.../wal/reader/IgniteWalIteratorFactory.java | 72 +-
.../wal/reader/StandaloneGridKernalContext.java | 6 +
.../reader/StandaloneWalRecordsIterator.java | 131 +-
.../persistence/wal/record/RecordTypes.java | 3 +
.../wal/serializer/RecordDataV1Serializer.java | 82 +
.../query/GridCacheDistributedQueryManager.java | 14 +-
.../cache/query/GridCacheQueryAdapter.java | 174 +-
.../cache/query/GridCacheQueryManager.java | 32 +-
.../cache/query/GridCacheQueryRequest.java | 31 +-
.../query/GridCacheQueryResponseEntry.java | 5 +-
.../cache/query/IgniteQueryErrorCode.java | 28 +
.../cache/query/QueryEntityTypeDescriptor.java | 39 +-
.../cache/query/SqlFieldsQueryEx.java | 39 +
.../continuous/CacheContinuousQueryManager.java | 8 +-
.../cache/transactions/IgniteInternalTx.java | 24 +-
.../transactions/IgniteTransactionsImpl.java | 4 +-
.../cache/transactions/IgniteTxAdapter.java | 247 +-
.../cache/transactions/IgniteTxEntry.java | 6 +
.../cache/transactions/IgniteTxHandler.java | 108 +-
.../IgniteTxImplicitSingleStateImpl.java | 16 +-
.../transactions/IgniteTxLocalAdapter.java | 332 +-
.../cache/transactions/IgniteTxLocalEx.java | 8 +
.../cache/transactions/IgniteTxLocalState.java | 16 +
.../transactions/IgniteTxLocalStateAdapter.java | 26 +
.../cache/transactions/IgniteTxManager.java | 13 +
.../IgniteTxRemoteSingleStateImpl.java | 5 +
.../IgniteTxRemoteStateAdapter.java | 21 +-
.../transactions/IgniteTxRemoteStateImpl.java | 10 +
.../cache/transactions/IgniteTxState.java | 16 +-
.../cache/transactions/IgniteTxStateImpl.java | 26 +-
.../cache/transactions/TxCounters.java | 82 +
.../cache/tree/AbstractDataInnerIO.java | 112 +-
.../cache/tree/AbstractDataLeafIO.java | 125 +-
.../cache/tree/CacheDataRowStore.java | 35 +-
.../processors/cache/tree/CacheDataTree.java | 82 +-
.../cache/tree/CacheIdAwareDataInnerIO.java | 8 +-
.../cache/tree/CacheIdAwareDataLeafIO.java | 8 +-
.../processors/cache/tree/DataInnerIO.java | 11 -
.../processors/cache/tree/DataLeafIO.java | 11 -
.../internal/processors/cache/tree/DataRow.java | 28 +-
.../cache/tree/PendingEntriesTree.java | 2 +-
.../processors/cache/tree/PendingRow.java | 3 +-
.../processors/cache/tree/RowLinkIO.java | 67 +-
.../processors/cache/tree/SearchRow.java | 31 +
.../mvcc/data/MvccCacheIdAwareDataInnerIO.java | 68 +
.../mvcc/data/MvccCacheIdAwareDataLeafIO.java | 89 +
.../cache/tree/mvcc/data/MvccDataInnerIO.java | 71 +
.../cache/tree/mvcc/data/MvccDataLeafIO.java | 89 +
.../cache/tree/mvcc/data/MvccDataRow.java | 263 ++
.../cache/tree/mvcc/data/MvccUpdateDataRow.java | 485 +++
.../tree/mvcc/data/MvccUpdateDataRowNative.java | 240 ++
.../cache/tree/mvcc/data/MvccUpdateResult.java | 43 +
.../cache/tree/mvcc/data/ResultType.java | 34 +
.../mvcc/search/MvccFirstRowTreeClosure.java | 76 +
.../mvcc/search/MvccLinkAwareSearchRow.java | 48 +
.../tree/mvcc/search/MvccMaxSearchRow.java | 55 +
.../tree/mvcc/search/MvccMinSearchRow.java | 55 +
.../cache/tree/mvcc/search/MvccSearchRow.java | 70 +
.../tree/mvcc/search/MvccSnapshotSearchRow.java | 113 +
.../cache/tree/mvcc/search/MvccTreeClosure.java | 29 +
.../continuous/GridContinuousProcessor.java | 33 +-
.../datastreamer/DataStreamerImpl.java | 68 +-
.../datastructures/DataStructuresProcessor.java | 5 +-
.../datastructures/GridSetQueryPredicate.java | 4 +-
.../odbc/ClientListenerNioListener.java | 4 +-
.../internal/processors/odbc/SqlStateCode.java | 3 +
.../odbc/jdbc/JdbcBatchExecuteRequest.java | 55 +-
.../odbc/jdbc/JdbcBatchExecuteResult.java | 11 +-
.../odbc/jdbc/JdbcBulkLoadAckResult.java | 11 +-
.../odbc/jdbc/JdbcBulkLoadBatchRequest.java | 11 +-
.../processors/odbc/jdbc/JdbcColumnMeta.java | 7 +-
.../processors/odbc/jdbc/JdbcColumnMetaV2.java | 11 +-
.../processors/odbc/jdbc/JdbcColumnMetaV3.java | 11 +-
.../processors/odbc/jdbc/JdbcColumnMetaV4.java | 11 +-
.../odbc/jdbc/JdbcConnectionContext.java | 55 +-
.../processors/odbc/jdbc/JdbcIndexMeta.java | 7 +-
.../processors/odbc/jdbc/JdbcMessageParser.java | 13 +-
.../odbc/jdbc/JdbcMetaColumnsRequest.java | 11 +-
.../odbc/jdbc/JdbcMetaColumnsResult.java | 15 +-
.../odbc/jdbc/JdbcMetaIndexesRequest.java | 11 +-
.../odbc/jdbc/JdbcMetaIndexesResult.java | 15 +-
.../odbc/jdbc/JdbcMetaParamsRequest.java | 11 +-
.../odbc/jdbc/JdbcMetaParamsResult.java | 15 +-
.../odbc/jdbc/JdbcMetaPrimaryKeysRequest.java | 11 +-
.../odbc/jdbc/JdbcMetaPrimaryKeysResult.java | 15 +-
.../odbc/jdbc/JdbcMetaSchemasRequest.java | 11 +-
.../odbc/jdbc/JdbcMetaSchemasResult.java | 11 +-
.../odbc/jdbc/JdbcMetaTablesRequest.java | 11 +-
.../odbc/jdbc/JdbcMetaTablesResult.java | 15 +-
.../jdbc/JdbcOrderedBatchExecuteRequest.java | 14 +-
.../jdbc/JdbcOrderedBatchExecuteResult.java | 11 +-
.../processors/odbc/jdbc/JdbcParameterMeta.java | 7 +-
.../odbc/jdbc/JdbcPrimaryKeyMeta.java | 7 +-
.../processors/odbc/jdbc/JdbcQuery.java | 7 +-
.../odbc/jdbc/JdbcQueryCloseRequest.java | 11 +-
...dbcQueryExecuteMultipleStatementsResult.java | 15 +-
.../odbc/jdbc/JdbcQueryExecuteRequest.java | 36 +-
.../odbc/jdbc/JdbcQueryExecuteResult.java | 11 +-
.../odbc/jdbc/JdbcQueryFetchRequest.java | 11 +-
.../odbc/jdbc/JdbcQueryFetchResult.java | 11 +-
.../odbc/jdbc/JdbcQueryMetadataRequest.java | 11 +-
.../odbc/jdbc/JdbcQueryMetadataResult.java | 15 +-
.../odbc/jdbc/JdbcRawBinarylizable.java | 8 +-
.../processors/odbc/jdbc/JdbcRequest.java | 13 +-
.../odbc/jdbc/JdbcRequestHandler.java | 72 +-
.../odbc/jdbc/JdbcRequestHandlerWorker.java | 131 +
.../processors/odbc/jdbc/JdbcResponse.java | 11 +-
.../processors/odbc/jdbc/JdbcResult.java | 12 +-
.../processors/odbc/jdbc/JdbcResultInfo.java | 7 +-
.../processors/odbc/jdbc/JdbcTableMeta.java | 7 +-
.../processors/odbc/odbc/OdbcColumnMeta.java | 37 +-
.../odbc/odbc/OdbcConnectionContext.java | 25 +-
.../processors/odbc/odbc/OdbcMessageParser.java | 16 +-
.../odbc/odbc/OdbcQueryExecuteBatchRequest.java | 15 +-
.../odbc/odbc/OdbcQueryExecuteRequest.java | 14 +-
.../processors/odbc/odbc/OdbcQueryResults.java | 10 +-
.../odbc/odbc/OdbcRequestHandler.java | 91 +-
.../odbc/odbc/OdbcRequestHandlerWorker.java | 131 +
.../processors/odbc/odbc/OdbcResultSet.java | 16 +-
.../platform/PlatformContextImpl.java | 2 +-
.../platform/PlatformProcessorImpl.java | 9 +-
.../platform/cache/PlatformCache.java | 4 +-
.../client/ClientConnectionContext.java | 17 +-
.../platform/client/ClientMessageParser.java | 16 +-
.../binary/ClientBinaryTypePutRequest.java | 2 +-
.../ClientCacheConfigurationSerializer.java | 11 +-
...ientCacheCreateWithConfigurationRequest.java | 6 +-
.../ClientCacheGetConfigurationRequest.java | 11 +-
.../ClientCacheGetConfigurationResponse.java | 11 +-
...acheGetOrCreateWithConfigurationRequest.java | 6 +-
.../platform/compute/PlatformCompute.java | 10 +
.../PlatformDotNetConfigurationClosure.java | 4 +-
.../utils/PlatformConfigurationUtils.java | 102 +-
.../processors/query/EnlistOperation.java | 91 +
.../query/GridQueryFieldMetadata.java | 16 +-
.../processors/query/GridQueryIndexing.java | 41 +-
.../processors/query/GridQueryProcessor.java | 89 +-
.../internal/processors/query/NestedTxMode.java | 60 +
.../internal/processors/query/QueryField.java | 2 +
.../query/QueryTypeDescriptorImpl.java | 71 +-
.../internal/processors/query/QueryUtils.java | 103 +-
.../processors/query/UpdateSourceIterator.java | 89 +
.../twostep/messages/GridQueryFailResponse.java | 2 +-
.../messages/GridQueryNextPageResponse.java | 65 +-
.../schema/SchemaIndexCacheVisitorImpl.java | 2 +-
.../processors/rest/GridRestProcessor.java | 23 +-
.../GridInternalSubscriptionProcessor.java | 18 +
.../task/GridTaskThreadContextKey.java | 3 +
.../processors/task/GridTaskWorker.java | 7 +-
.../timeout/GridTimeoutProcessor.java | 94 +
.../apache/ignite/internal/sql/SqlKeyword.java | 20 +
.../apache/ignite/internal/sql/SqlParser.java | 85 +-
.../ignite/internal/sql/SqlParserUtils.java | 14 +
.../sql/command/SqlBeginTransactionCommand.java | 46 +
.../command/SqlCommitTransactionCommand.java | 46 +
.../command/SqlRollbackTransactionCommand.java | 46 +
.../IgniteTxMvccVersionCheckedException.java | 47 +
.../ignite/internal/util/IgniteUtils.java | 16 +
.../util/future/GridCompoundFuture.java | 4 +-
.../util/snaptree/CopyOnWriteManager.java | 390 --
.../ignite/internal/util/snaptree/Epoch.java | 131 -
.../internal/util/snaptree/EpochNode.java | 433 --
.../internal/util/snaptree/SnapTreeMap.java | 2917 --------------
.../internal/util/snaptree/package-info.java | 22 -
.../util/tostring/GridToStringBuilder.java | 139 +-
.../VisorCacheConfigurationCollectorTask.java | 1 -
.../cache/VisorCacheLostPartitionsTask.java | 5 +-
.../visor/cache/VisorCacheModifyTask.java | 14 +-
.../visor/cache/VisorCacheModifyTaskArg.java | 14 +-
.../visor/cache/VisorCacheModifyTaskResult.java | 28 +-
.../cache/VisorCacheNamesCollectorTask.java | 90 +
.../VisorCacheNamesCollectorTaskResult.java | 88 +
.../visor/cache/VisorCachePartitionsTask.java | 2 +-
.../VisorCacheResetLostPartitionsTask.java | 2 +-
.../visor/compute/VisorGatewayTask.java | 2 +-
.../internal/visor/file/VisorFileBlock.java | 2 +-
.../visor/node/VisorNodeDataCollectorJob.java | 20 +-
.../visor/node/VisorNodeDataCollectorTask.java | 2 +-
.../node/VisorNodeDataCollectorTaskArg.java | 49 +-
.../visor/service/VisorServiceTask.java | 2 +-
.../ignite/internal/visor/tx/VisorTxInfo.java | 58 +-
.../ignite/internal/visor/tx/VisorTxTask.java | 20 +-
.../org/apache/ignite/spi/IgniteSpiAdapter.java | 2 +-
.../communication/tcp/TcpCommunicationSpi.java | 73 +-
.../ignite/spi/discovery/tcp/ServerImpl.java | 22 +-
.../spi/discovery/tcp/TcpDiscoverySpi.java | 8 +-
.../tcp/internal/TcpDiscoveryNode.java | 24 +-
.../tcp/internal/TcpDiscoveryNodesRing.java | 8 +-
.../ignite/ssl/DelegatingSSLContextSpi.java | 103 +
.../apache/ignite/ssl/SSLContextWrapper.java | 31 +
.../ssl/SSLServerSocketFactoryWrapper.java | 80 +
.../ignite/ssl/SSLSocketFactoryWrapper.java | 113 +
.../apache/ignite/ssl/SslContextFactory.java | 51 +
.../org/apache/ignite/thread/IgniteThread.java | 45 +-
.../resources/META-INF/classnames.properties | 106 +-
modules/core/src/test/config/tests.properties | 5 -
.../client/ClientCacheConfigurationTest.java | 7 +
.../ignite/client/ClientConfigurationTest.java | 7 +
.../apache/ignite/client/FunctionalTest.java | 7 +
.../apache/ignite/client/IgniteBinaryTest.java | 11 +-
.../java/org/apache/ignite/client/LoadTest.java | 10 +-
.../apache/ignite/client/ReliabilityTest.java | 12 +-
.../apache/ignite/client/SslParametersTest.java | 329 ++
.../ignite/custom/DummyEventFilterFactory.java | 10 +-
.../failure/AccountTransferTransactionTest.java | 331 --
.../ignite/internal/ClusterMetricsSelfTest.java | 12 +-
.../internal/ClusterNodeMetricsSelfTest.java | 14 +-
.../apache/ignite/internal/GridStartupMain.java | 58 -
.../apache/ignite/internal/GridStartupTest.java | 71 -
.../internal/GridTaskResultCacheSelfTest.java | 26 +-
.../internal/TestRecordingCommunicationSpi.java | 26 +-
.../binary/BinaryMarshallerSelfTest.java | 35 +-
.../binary/TestCachingMetadataHandler.java | 2 +-
.../FullyConnectedComponentSearcherTest.java | 7 +
.../GridDiscoveryManagerAttributesSelfTest.java | 53 +-
...ataRegistrationInsideEntryProcessorTest.java | 141 +
.../cache/CacheEntryProcessorCopySelfTest.java | 2 +-
.../cache/CacheStopAndDestroySelfTest.java | 68 +
.../cache/GridCacheAbstractFullApiSelfTest.java | 2 +-
.../processors/cache/GridCacheTestEntryEx.java | 78 +-
.../IgniteCacheConfigVariationsFullApiTest.java | 2 +-
.../cache/IgniteCacheInvokeAbstractTest.java | 150 +
...IgniteClientCacheInitializationFailTest.java | 20 +-
...gniteTopologyValidatorAbstractCacheTest.java | 2 +-
...rtitionsExchangeCoordinatorFailoverTest.java | 155 +
...eAbstractDataStructuresFailoverSelfTest.java | 16 +-
.../GridCacheSetAbstractSelfTest.java | 4 -
.../GridCacheSetFailoverAbstractSelfTest.java | 6 +-
...chePartitionedAtomicSetFailoverSelfTest.java | 2 +-
...rtitionedDataStructuresFailoverSelfTest.java | 14 +
.../IgnitePartitionedSetNoBackupsSelfTest.java | 2 +-
...eplicatedDataStructuresFailoverSelfTest.java | 22 +-
.../CacheDataLossOnPartitionMoveTest.java | 296 ++
.../distributed/CacheExchangeMergeTest.java | 2 -
.../CacheLateAffinityAssignmentTest.java | 2 +-
.../CacheRentingStateRepairTest.java | 189 +
...niteCacheClientNodeChangingTopologyTest.java | 3 +-
.../IgniteCachePartitionLossPolicySelfTest.java | 84 +-
.../dht/GridCacheTxNodeFailureSelfTest.java | 2 +-
.../dht/IgniteCacheStartWithLoadTest.java | 165 +
.../GridCacheRebalancingSyncSelfTest.java | 2 +-
.../cache/eviction/DhtAndNearEvictionTest.java | 189 +
...vccAbstractBasicCoordinatorFailoverTest.java | 681 ++++
...acheMvccAbstractCoordinatorFailoverTest.java | 148 +
.../mvcc/CacheMvccAbstractFeatureTest.java | 299 ++
.../cache/mvcc/CacheMvccAbstractTest.java | 2302 +++++++++++
.../cache/mvcc/CacheMvccClusterRestartTest.java | 178 +
.../CacheMvccConfigurationValidationTest.java | 191 +
...ccIteratorWithConcurrentTransactionTest.java | 64 +
...calEntriesWithConcurrentTransactionTest.java | 65 +
.../mvcc/CacheMvccOperationChecksTest.java | 201 +
...eMvccPartitionedCoordinatorFailoverTest.java | 144 +
.../cache/mvcc/CacheMvccProcessorTest.java | 97 +
...heMvccReplicatedCoordinatorFailoverTest.java | 30 +
...cScanQueryWithConcurrentTransactionTest.java | 76 +
...heMvccSizeWithConcurrentTransactionTest.java | 52 +
.../cache/mvcc/CacheMvccTransactionsTest.java | 3683 ++++++++++++++++++
.../cache/mvcc/CacheMvccVacuumTest.java | 158 +
...cheObjectBinaryProcessorOnDiscoveryTest.java | 125 +
.../IgnitePdsCorruptedStoreTest.java | 1 -
.../persistence/IgnitePdsTaskCancelingTest.java | 6 +-
...IgnitePersistentStoreDataStructuresTest.java | 45 +-
.../MemoryPolicyInitializationTest.java | 12 +-
.../db/IgnitePdsDataRegionMetricsTest.java | 59 +-
.../db/IgnitePdsReserveWalSegmentsTest.java | 188 +
.../db/IgnitePdsUnusedWalSegmentsTest.java | 198 -
.../file/IgnitePdsDiskErrorsRecoveringTest.java | 5 +-
.../IgniteNodeStoppedDuringDisableWALTest.java | 5 +-
...IsReachedDuringIterationOverArchiveTest.java | 5 +-
.../db/wal/IgniteWalFormatFileFailoverTest.java | 2 +-
.../db/wal/IgniteWalRebalanceTest.java | 185 +-
.../db/wal/IgniteWalRecoveryTest.java | 1906 ---------
.../IgniteWalRecoveryWithCompactionTest.java | 33 -
.../persistence/db/wal/WalCompactionTest.java | 9 +-
.../db/wal/WalDeletionArchiveAbstractTest.java | 277 ++
.../db/wal/WalDeletionArchiveFsyncTest.java | 31 +
.../db/wal/WalDeletionArchiveLogOnlyTest.java | 31 +
.../db/wal/WalRecoveryTxLogicalRecordsTest.java | 44 -
.../db/wal/reader/IgniteWalReaderTest.java | 245 +-
.../db/wal/reader/MockWalIteratorFactory.java | 1 +
.../pagemem/FullPageIdTableTest.java | 7 +
...gnitePageMemReplaceDelayedWriteUnitTest.java | 7 +
.../pagemem/IgniteThrottlingUnitTest.java | 7 +
.../pagemem/NoOpPageStoreManager.java | 7 +
.../persistence/pagemem/NoOpWALManager.java | 15 +-
.../RobinHoodBackwardShiftHashMapTest.java | 6 +
.../StandaloneWalRecordsIteratorTest.java | 216 +
.../ContinuousQueryMarshallerTest.java | 168 +
.../ContinuousQueryPeerClassLoadingTest.java | 4 +-
.../AbstractDeadlockDetectionTest.java | 14 +
.../AbstractTransactionIntergrityTest.java | 595 +++
...IntegrityWithPrimaryIndexCorruptionTest.java | 238 ++
...ctionIntegrityWithSystemWorkerDeathTest.java | 106 +
.../TxOptimisticDeadlockDetectionTest.java | 19 +-
.../TxPessimisticDeadlockDetectionTest.java | 10 +-
.../cache/transactions/TxRollbackAsyncTest.java | 148 +-
.../transactions/TxRollbackOnTimeoutTest.java | 225 +-
.../TxRollbackOnTopologyChangeTest.java | 16 -
.../processors/database/BPlusTreeSelfTest.java | 377 +-
.../database/CacheFreeListImplSelfTest.java | 55 +
.../database/DataRegionMetricsSelfTest.java | 24 +-
.../IgniteDbPutGetWithCacheStoreTest.java | 5 +-
.../DataStreamProcessorMvccSeflTest.java | 68 +
.../DataStreamProcessorMvccSelfTest.java | 70 +
.../DataStreamProcessorSelfTest.java | 24 +-
.../SqlParserTransactionalKeywordsSelfTest.java | 96 +
.../internal/util/IgniteDevOnlyLogTest.java | 137 +-
.../tostring/GridToStringBuilderSelfTest.java | 110 +-
.../IgniteOffheapReadWriteLockSelfTest.java | 34 +-
.../mapper/GridContinuousMapperLoadTest1.java | 51 -
.../mapper/GridContinuousMapperLoadTest2.java | 92 -
.../mapper/GridContinuousMapperTask1.java | 157 -
.../mapper/GridContinuousMapperTask2.java | 93 -
.../ignite/loadtests/mapper/TestObject.java | 57 -
.../tcp/TcpDiscoverySslParametersTest.java | 274 ++
.../ignite/testframework/GridTestUtils.java | 46 +-
.../testframework/junits/GridAbstractTest.java | 37 +-
.../junits/common/GridCommonAbstractTest.java | 159 +-
.../ignite/testsuites/IgniteBasicTestSuite.java | 5 +-
.../IgniteBasicWithPersistenceTestSuite.java | 5 +-
.../IgniteCacheEvictionSelfTestSuite.java | 3 +
.../testsuites/IgniteCacheMvccTestSuite.java | 67 +
.../ignite/testsuites/IgniteCacheTestSuite.java | 3 +
.../testsuites/IgniteCacheTestSuite4.java | 16 -
.../testsuites/IgniteCacheTestSuite6.java | 9 +-
.../testsuites/IgniteCacheTestSuite7.java | 13 +-
.../testsuites/IgniteCacheTestSuite9.java | 54 +
.../testsuites/IgniteKernalSelfTestSuite.java | 11 +-
.../testsuites/IgniteLangSelfTestSuite.java | 2 +-
.../testsuites/IgniteLostAndFoundTestSuite.java | 1 -
.../ignite/testsuites/IgnitePdsTestSuite.java | 6 +-
.../ignite/testsuites/IgnitePdsTestSuite2.java | 45 +-
.../ignite/testsuites/IgnitePdsTestSuite3.java | 49 +
.../ignite/testsuites/IgnitePdsTestSuite4.java | 63 +
.../IgniteSpiDiscoverySelfTestSuite.java | 6 +-
.../testsuites/IgniteUtilSelfTestSuite.java | 22 +-
.../ignite/util/GridCommandHandlerTest.java | 23 -
...dConcurrentLinkedDequeMultiThreadedTest.java | 5 +-
.../apache/ignite/util/GridIndexFillTest.java | 259 --
.../apache/ignite/util/GridRandomSelfTest.java | 6 +-
.../development/utils/IgniteWalConverter.java | 5 +-
.../query/h2/opt/GridH2SpatialIndex.java | 11 +-
.../impl/v2/HadoopV2JobResourceManager.java | 3 +-
...niteHadoopFileSystemClientBasedOpenTest.java | 2 +-
.../cache/hibernate/HibernateCacheProxy.java | 10 -
.../cache/query/GridCacheTwoStepQuery.java | 36 +
.../query/h2/DhtResultSetEnlistFuture.java | 63 +
.../query/h2/DmlStatementsProcessor.java | 352 +-
.../query/h2/H2ConnectionWrapper.java | 52 +-
.../processors/query/h2/H2FieldsIterator.java | 29 +-
.../processors/query/h2/H2KeyValueIterator.java | 2 +-
.../query/h2/H2ResultSetIterator.java | 47 +-
.../internal/processors/query/h2/H2Schema.java | 31 +-
.../processors/query/h2/H2SqlFieldMetadata.java | 28 +-
.../processors/query/h2/H2StatementCache.java | 78 +-
.../internal/processors/query/h2/H2Utils.java | 4 +-
.../processors/query/h2/IgniteH2Indexing.java | 1153 ++++--
.../query/h2/NearResultSetEnlistFuture.java | 62 +
.../query/h2/PreparedStatementEx.java | 45 +
.../query/h2/PreparedStatementExImpl.java | 648 +++
.../query/h2/RebuildIndexFromHashClosure.java | 47 +
.../query/h2/ResultSetEnlistFuture.java | 136 +
.../query/h2/ThreadLocalObjectPool.java | 103 +
.../query/h2/database/H2PkHashIndex.java | 108 +-
.../query/h2/database/H2RowFactory.java | 25 +-
.../processors/query/h2/database/H2Tree.java | 112 +-
.../query/h2/database/H2TreeFilterClosure.java | 109 +
.../query/h2/database/H2TreeIndex.java | 160 +-
.../h2/database/io/AbstractH2ExtrasInnerIO.java | 164 +
.../h2/database/io/AbstractH2ExtrasLeafIO.java | 161 +
.../query/h2/database/io/AbstractH2InnerIO.java | 74 +
.../query/h2/database/io/AbstractH2LeafIO.java | 76 +
.../query/h2/database/io/H2ExtrasInnerIO.java | 111 +-
.../query/h2/database/io/H2ExtrasLeafIO.java | 107 +-
.../query/h2/database/io/H2IOUtils.java | 94 +
.../query/h2/database/io/H2InnerIO.java | 41 +-
.../query/h2/database/io/H2LeafIO.java | 41 +-
.../h2/database/io/H2MvccExtrasInnerIO.java | 55 +
.../h2/database/io/H2MvccExtrasLeafIO.java | 54 +
.../query/h2/database/io/H2MvccInnerIO.java | 58 +
.../query/h2/database/io/H2MvccLeafIO.java | 58 +
.../query/h2/database/io/H2RowLinkIO.java | 34 +
.../query/h2/ddl/DdlStatementsProcessor.java | 180 +-
.../processors/query/h2/dml/DmlAstUtils.java | 12 +-
.../processors/query/h2/dml/FastUpdate.java | 17 +
.../processors/query/h2/dml/UpdatePlan.java | 233 ++
.../query/h2/dml/UpdatePlanBuilder.java | 118 +-
.../query/h2/opt/GridH2IndexBase.java | 36 +-
.../query/h2/opt/GridH2KeyRowOnheap.java | 11 +
.../query/h2/opt/GridH2KeyValueRowOnheap.java | 21 +-
.../query/h2/opt/GridH2MetaTable.java | 5 +
.../query/h2/opt/GridH2PlainRowFactory.java | 17 +-
.../query/h2/opt/GridH2QueryContext.java | 27 +-
.../processors/query/h2/opt/GridH2Row.java | 50 +-
.../query/h2/opt/GridH2RowDescriptor.java | 9 +-
.../query/h2/opt/GridH2SearchRow.java | 32 +
.../query/h2/opt/GridH2SearchRowAdapter.java | 27 +-
.../processors/query/h2/opt/GridH2Table.java | 8 +-
.../query/h2/sql/GridSqlQueryParser.java | 164 +-
.../query/h2/sql/GridSqlQuerySplitter.java | 18 +-
.../processors/query/h2/sql/GridSqlSelect.java | 20 +
.../processors/query/h2/sys/SqlSystemIndex.java | 5 +-
.../processors/query/h2/sys/SqlSystemTable.java | 4 +-
.../h2/sys/view/SqlAbstractLocalSystemView.java | 55 +-
.../query/h2/sys/view/SqlSystemView.java | 3 +-
.../h2/sys/view/SqlSystemViewBaselineNodes.java | 87 +
.../sys/view/SqlSystemViewNodeAttributes.java | 108 +
.../h2/sys/view/SqlSystemViewNodeMetrics.java | 210 +
.../query/h2/sys/view/SqlSystemViewNodes.java | 32 +-
.../query/h2/twostep/GridMapQueryExecutor.java | 522 ++-
.../h2/twostep/GridMergeIndexIterator.java | 17 +-
.../query/h2/twostep/GridMergeIndexSorted.java | 4 +-
.../h2/twostep/GridReduceQueryExecutor.java | 252 +-
.../query/h2/twostep/GridResultPage.java | 20 +-
.../query/h2/twostep/MapQueryResult.java | 2 +-
.../query/h2/twostep/MapQueryResults.java | 18 +-
.../query/h2/twostep/ReduceQueryRun.java | 152 +-
.../h2/twostep/msg/GridH2QueryRequest.java | 75 +-
.../msg/GridH2SelectForUpdateTxDetails.java | 299 ++
.../twostep/msg/GridH2ValueMessageFactory.java | 12 +-
.../apache/ignite/client/ClientTestSuite.java | 3 +-
.../ignite/client/FunctionalQueryTest.java | 7 +
.../ignite/client/IgniteBinaryQueryTest.java | 7 +
.../org/apache/ignite/client/SecurityTest.java | 7 +
...CacheScanPartitionQueryFallbackSelfTest.java | 62 +
.../cache/DdlTransactionSelfTest.java | 335 ++
.../IgniteCacheAbstractFieldsQuerySelfTest.java | 104 +-
.../cache/IgniteCacheAbstractQuerySelfTest.java | 54 +-
.../IgniteCacheQueryH2IndexingLeakTest.java | 10 +-
...ndexingCachePartitionLossPolicySelfTest.java | 139 +
...niteCacheDistributedQueryCancelSelfTest.java | 2 +-
.../IgniteCacheReplicatedQuerySelfTest.java | 27 -
.../cache/index/AbstractSchemaSelfTest.java | 102 +-
...ynamicColumnsAbstractConcurrentSelfTest.java | 13 +-
.../index/DynamicIndexAbstractSelfTest.java | 7 +
.../cache/index/H2ConnectionLeaksSelfTest.java | 8 +-
.../H2DynamicColumnsAbstractBasicSelfTest.java | 117 +
.../index/H2DynamicIndexAbstractSelfTest.java | 55 +-
.../cache/index/H2DynamicTableSelfTest.java | 136 +-
.../cache/index/IgniteDecimalSelfTest.java | 35 +-
.../index/SqlTransactionsComandsSelfTest.java | 83 +
...sactionsComandsWithMvccDisabledSelfTest.java | 83 +
...sactionsCommandsWithMvccEnabledSelfTest.java | 420 ++
.../cache/index/SqlTransactionsSelfTest.java | 421 ++
.../local/IgniteCacheLocalQuerySelfTest.java | 64 +-
...eMvccAbstractSqlCoordinatorFailoverTest.java | 138 +
.../mvcc/CacheMvccBackupsAbstractTest.java | 808 ++++
.../cache/mvcc/CacheMvccBulkLoadTest.java | 123 +
.../cache/mvcc/CacheMvccDmlSimpleTest.java | 200 +
...eratorWithConcurrentJdbcTransactionTest.java | 28 +
...ntriesWithConcurrentJdbcTransactionTest.java | 29 +
.../mvcc/CacheMvccPartitionedBackupsTest.java | 30 +
...MvccPartitionedSelectForUpdateQueryTest.java | 44 +
...ccPartitionedSqlCoordinatorFailoverTest.java | 143 +
.../CacheMvccPartitionedSqlQueriesTest.java | 30 +
.../CacheMvccPartitionedSqlTxQueriesTest.java | 30 +
...cPartitionedSqlTxQueriesWithReducerTest.java | 30 +
.../mvcc/CacheMvccReplicatedBackupsTest.java | 30 +
...eMvccReplicatedSelectForUpdateQueryTest.java | 30 +
...vccReplicatedSqlCoordinatorFailoverTest.java | 30 +
.../mvcc/CacheMvccReplicatedSqlQueriesTest.java | 30 +
.../CacheMvccReplicatedSqlTxQueriesTest.java | 242 ++
...ccReplicatedSqlTxQueriesWithReducerTest.java | 30 +
...nQueryWithConcurrentJdbcTransactionTest.java | 29 +
...cheMvccSelectForUpdateQueryAbstractTest.java | 374 ++
.../cache/mvcc/CacheMvccSizeTest.java | 488 +++
...ccSizeWithConcurrentJdbcTransactionTest.java | 28 +
.../mvcc/CacheMvccSqlQueriesAbstractTest.java | 1613 ++++++++
.../mvcc/CacheMvccSqlTxQueriesAbstractTest.java | 1832 +++++++++
...MvccSqlTxQueriesWithReducerAbstractTest.java | 829 ++++
.../mvcc/CacheMvccStreamingInsertTest.java | 122 +
.../db/wal/IgniteWalRecoveryTest.java | 1911 +++++++++
.../IgniteWalRecoveryWithCompactionTest.java | 33 +
.../IgniteTwoRegionsRebuildIndexTest.java | 124 +
.../query/SqlIllegalSchemaSelfTest.java | 187 +
.../query/SqlSystemViewsSelfTest.java | 360 +-
.../query/h2/GridIndexRebuildSelfTest.java | 240 ++
...GridIndexRebuildWithMvccEnabledSelfTest.java | 126 +
.../h2/GridIndexingSpiAbstractSelfTest.java | 4 +-
...H2ResultSetIteratorNullifyOnEndSelfTest.java | 420 ++
.../query/h2/H2StatementCacheSelfTest.java | 83 +
.../query/h2/PreparedStatementExSelfTest.java | 61 +
.../query/h2/ThreadLocalObjectPoolSelfTest.java | 113 +
.../query/h2/sql/GridQueryParsingTest.java | 1 +
.../h2/twostep/CacheQueryMemoryLeakTest.java | 159 +
...sappearedCacheCauseRetryMessageSelfTest.java | 134 +
...appearedCacheWasNotFoundMessageSelfTest.java | 123 +
.../query/h2/twostep/JoinSqlTestHelper.java | 163 +
.../NonCollocatedRetryMessageSelfTest.java | 146 +
.../h2/twostep/RetryCauseMessageSelfTest.java | 417 ++
...ePartitionedAtomicColumnConstraintsTest.java | 398 ++
...ionedTransactionalColumnConstraintsTest.java | 30 +
...heReplicatedAtomicColumnConstraintsTest.java | 32 +
...catedTransactionalColumnConstraintsTest.java | 30 +
.../sql/IgniteSQLColumnConstraintsTest.java | 143 +
.../loadtests/h2indexing/GridTestEntity.java | 67 -
.../loadtests/h2indexing/GridTreeBenchmark.java | 280 --
.../testsuites/IgniteCacheMvccSqlTestSuite.java | 90 +
.../IgniteCacheQuerySelfTestSuite.java | 56 +-
.../IgniteCacheQuerySelfTestSuite2.java | 14 +-
.../IgniteCacheQuerySelfTestSuite3.java | 2 +
.../IgnitePdsWithIndexingCoreTestSuite.java | 2 +
.../IgnitePdsWithIndexingTestSuite.java | 2 +
.../processors/cache/mvcc/mvcc_person.csv | 2 +
.../cache/mvcc/mvcc_person_broken.csv | 2 +
.../processors/cache/jta/CacheJtaManager.java | 1 +
.../config/az/ignite-account-role.yaml | 30 +
.../kubernetes/config/az/ignite-deployment.yaml | 47 +
.../kubernetes/config/az/ignite-namespace.yaml | 19 +
.../az/ignite-persistence-storage-class.yaml | 25 +
.../config/az/ignite-role-binding.yaml | 27 +
.../config/az/ignite-service-account.yaml | 20 +
.../kubernetes/config/az/ignite-service.yaml | 38 +
.../config/az/ignite-stateful-set.yaml | 74 +
.../config/az/ignite-wal-storage-class.yaml | 25 +
.../config/gce/ignite-account-role.yaml | 30 +
.../config/gce/ignite-deployment.yaml | 47 +
.../kubernetes/config/gce/ignite-namespace.yaml | 19 +
.../gce/ignite-persistence-storage-class.yaml | 26 +
.../config/gce/ignite-role-binding.yaml | 27 +
.../config/gce/ignite-service-account.yaml | 20 +
.../kubernetes/config/gce/ignite-service.yaml | 38 +
.../config/gce/ignite-stateful-set.yaml | 74 +
.../config/gce/ignite-wal-storage-class.yaml | 26 +
modules/mesos/pom.xml | 2 +-
.../apache/ignite/mesos/IgniteFramework.java | 3 +-
.../ignite/mesos/IgniteSchedulerSelfTest.java | 11 +
modules/ml/pom.xml | 13 +-
.../clustering/kmeans/ClusterizationModel.java | 4 +-
.../ml/clustering/kmeans/KMeansModel.java | 4 +-
.../ml/clustering/kmeans/KMeansTrainer.java | 134 +-
.../ml/composition/BaggingModelTrainer.java | 20 +
.../ml/composition/ModelsComposition.java | 10 +-
.../ml/composition/ModelsCompositionFormat.java | 61 +
.../boosting/GDBBinaryClassifierTrainer.java | 59 +-
.../boosting/GDBLearningStrategy.java | 115 +-
.../boosting/GDBRegressionTrainer.java | 8 +-
.../ml/composition/boosting/GDBTrainer.java | 119 +-
.../LossGradientPerPredictionFunctions.java | 33 -
.../convergence/ConvergenceChecker.java | 140 +
.../convergence/ConvergenceCheckerFactory.java | 58 +
.../mean/MeanAbsValueConvergenceChecker.java | 116 +
.../MeanAbsValueConvergenceCheckerFactory.java | 47 +
.../boosting/convergence/mean/package-info.java | 22 +
.../MedianOfMedianConvergenceChecker.java | 126 +
...MedianOfMedianConvergenceCheckerFactory.java | 47 +
.../convergence/median/package-info.java | 22 +
.../boosting/convergence/package-info.java | 24 +
.../simple/ConvergenceCheckerStub.java | 79 +
.../simple/ConvergenceCheckerStubFactory.java | 48 +
.../convergence/simple/package-info.java | 24 +
.../ml/composition/boosting/loss/LogLoss.java | 36 +
.../ml/composition/boosting/loss/Loss.java | 45 +
.../composition/boosting/loss/SquaredError.java | 36 +
.../composition/boosting/loss/package-info.java | 22 +
.../WeightedPredictionsAggregator.java | 12 +-
.../ignite/ml/dataset/feature/BucketMeta.java | 88 +
.../dataset/feature/DistributionComputer.java | 32 +
.../ignite/ml/dataset/feature/FeatureMeta.java | 65 +
.../ignite/ml/dataset/feature/Histogram.java | 64 +
.../ml/dataset/feature/ObjectHistogram.java | 129 +
.../ignite/ml/dataset/feature/package-info.java | 24 +
.../BootstrappedDatasetBuilder.java | 90 +
.../BootstrappedDatasetPartition.java | 68 +
.../impl/bootstrapping/BootstrappedVector.java | 87 +
.../impl/bootstrapping/package-info.java | 23 +
.../FeatureMatrixWithLabelsOnHeapData.java | 57 +
...eatureMatrixWithLabelsOnHeapDataBuilder.java | 76 +
.../parallelism/ParallelismStrategy.java | 15 +-
.../java/org/apache/ignite/ml/knn/KNNUtils.java | 8 +-
.../ignite/ml/knn/NNClassificationModel.java | 233 ++
.../ml/knn/ann/ANNClassificationModel.java | 208 +
.../ml/knn/ann/ANNClassificationTrainer.java | 382 ++
.../ignite/ml/knn/ann/ANNModelFormat.java | 90 +
.../apache/ignite/ml/knn/ann/ProbableLabel.java | 39 +
.../apache/ignite/ml/knn/ann/package-info.java | 22 +
.../classification/KNNClassificationModel.java | 228 +-
.../KNNClassificationTrainer.java | 20 +-
.../ml/knn/classification/KNNModelFormat.java | 14 +-
.../ml/knn/classification/KNNStrategy.java | 29 -
.../ml/knn/classification/NNStrategy.java | 29 +
.../ml/knn/regression/KNNRegressionModel.java | 6 +-
.../ml/knn/regression/KNNRegressionTrainer.java | 19 +-
.../ml/math/isolve/lsqr/AbstractLSQR.java | 6 +-
.../ignite/ml/math/isolve/lsqr/LSQROnHeap.java | 10 +-
.../org/apache/ignite/ml/nn/MLPTrainer.java | 186 +-
.../ml/preprocessing/PreprocessingTrainer.java | 3 +-
.../binarization/BinarizationPreprocessor.java | 4 +-
.../binarization/BinarizationTrainer.java | 9 +-
.../preprocessing/encoding/EncoderTrainer.java | 11 +
.../linear/LinearRegressionLSQRTrainer.java | 27 +-
.../linear/LinearRegressionSGDTrainer.java | 159 +-
.../binomial/LogisticRegressionSGDTrainer.java | 155 +-
.../LogRegressionMultiClassModel.java | 9 +
.../LogRegressionMultiClassTrainer.java | 76 +-
.../ignite/ml/selection/cv/CrossValidation.java | 2 +-
.../ml/selection/paramgrid/ParamGrid.java | 14 +-
.../ignite/ml/structures/LabeledDataset.java | 220 --
.../structures/LabeledDatasetTestTrainPair.java | 114 -
.../ignite/ml/structures/LabeledVectorSet.java | 220 ++
.../LabeledVectorSetTestTrainPair.java | 114 +
...abeledDatasetPartitionDataBuilderOnHeap.java | 12 +-
.../preprocessing/LabeledDatasetLoader.java | 10 +-
.../SVMLinearBinaryClassificationTrainer.java | 130 +-
.../SVMLinearMultiClassClassificationModel.java | 9 +
...VMLinearMultiClassClassificationTrainer.java | 134 +-
.../ignite/ml/trainers/DatasetTrainer.java | 166 +
.../org/apache/ignite/ml/tree/DecisionTree.java | 27 +-
.../tree/DecisionTreeClassificationTrainer.java | 4 +-
.../ml/tree/DecisionTreeRegressionTrainer.java | 10 +-
.../GDBBinaryClassifierOnTreesTrainer.java | 73 +-
.../boosting/GDBOnTreesLearningStrategy.java | 24 +-
.../boosting/GDBRegressionOnTreesTrainer.java | 67 +-
.../ignite/ml/tree/data/DecisionTreeData.java | 36 +-
.../RandomForestClassifierTrainer.java | 121 +-
.../RandomForestRegressionTrainer.java | 82 +-
.../tree/randomforest/RandomForestTrainer.java | 396 +-
.../data/FeaturesCountSelectionStrategies.java | 47 +
.../ml/tree/randomforest/data/NodeId.java | 55 +
.../ml/tree/randomforest/data/NodeSplit.java | 79 +
.../ml/tree/randomforest/data/TreeNode.java | 200 +
.../ml/tree/randomforest/data/TreeRoot.java | 86 +
.../data/impurity/GiniHistogram.java | 225 ++
.../data/impurity/GiniHistogramsComputer.java | 46 +
.../data/impurity/ImpurityComputer.java | 38 +
.../data/impurity/ImpurityHistogram.java | 74 +
.../impurity/ImpurityHistogramsComputer.java | 211 +
.../data/impurity/MSEHistogram.java | 235 ++
.../data/impurity/MSEHistogramComputer.java | 33 +
.../data/impurity/package-info.java | 22 +
.../ml/tree/randomforest/data/package-info.java | 22 +
.../ClassifierLeafValuesComputer.java | 82 +
.../data/statistics/LeafValuesComputer.java | 158 +
.../data/statistics/MeanValueStatistic.java | 72 +
.../NormalDistributionStatistics.java | 112 +
.../NormalDistributionStatisticsComputer.java | 119 +
.../RegressionLeafValuesComputer.java | 62 +
.../data/statistics/package-info.java | 22 +
.../org/apache/ignite/ml/IgniteMLTestSuite.java | 1 -
.../org/apache/ignite/ml/LocalModelsTest.java | 190 -
.../ignite/ml/clustering/KMeansModelTest.java | 4 +-
.../ignite/ml/clustering/KMeansTrainerTest.java | 74 +-
.../ignite/ml/common/CollectionsTest.java | 16 +-
.../ignite/ml/common/CommonTestSuite.java | 4 +-
.../ignite/ml/common/LocalModelsTest.java | 264 ++
.../org/apache/ignite/ml/common/ModelTest.java | 45 +
.../apache/ignite/ml/common/TrainerTest.java | 1161 ++++++
.../ml/composition/boosting/GDBTrainerTest.java | 107 +-
.../convergence/ConvergenceCheckerTest.java | 82 +
.../MeanAbsValueConvergenceCheckerTest.java | 73 +
.../MedianOfMedianConvergenceCheckerTest.java | 57 +
.../WeightedPredictionsAggregatorTest.java | 19 +-
.../ignite/ml/dataset/DatasetTestSuite.java | 4 +-
.../ml/dataset/feature/ObjectHistogramTest.java | 170 +
.../impl/local/LocalDatasetBuilderTest.java | 27 +-
.../dataset/primitive/DatasetWrapperTest.java | 48 +-
.../ml/environment/EnvironmentTestSuite.java | 3 +-
.../ml/environment/LearningEnvironmentTest.java | 181 +-
.../ignite/ml/environment/PromiseTest.java | 79 +
.../ignite/ml/knn/ANNClassificationTest.java | 119 +
.../ignite/ml/knn/KNNClassificationTest.java | 63 +-
.../apache/ignite/ml/knn/KNNRegressionTest.java | 50 +-
.../org/apache/ignite/ml/knn/KNNTestSuite.java | 3 +-
.../ignite/ml/knn/LabeledDatasetHelper.java | 4 +-
.../ignite/ml/knn/LabeledDatasetTest.java | 294 --
.../ignite/ml/knn/LabeledVectorSetTest.java | 294 ++
.../java/org/apache/ignite/ml/nn/MLPTest.java | 2 +-
.../org/apache/ignite/ml/nn/MLPTrainerTest.java | 64 +
.../binarization/BinarizationTrainerTest.java | 6 +-
.../linear/LinearRegressionLSQRTrainerTest.java | 97 +-
.../linear/LinearRegressionSGDTrainerTest.java | 109 +-
.../logistic/LogRegMultiClassTrainerTest.java | 113 +-
.../LogisticRegressionSGDTrainerTest.java | 104 +-
.../ml/selection/cv/CrossValidationTest.java | 26 +-
.../scoring/evaluator/EvaluatorTest.java | 387 +-
.../ml/svm/SVMBinaryTrainerIntegrationTest.java | 102 -
.../ignite/ml/svm/SVMBinaryTrainerTest.java | 82 +-
.../ignite/ml/svm/SVMMultiClassTrainerTest.java | 87 +-
.../org/apache/ignite/ml/svm/SVMTestSuite.java | 1 -
.../tree/DecisionTreeRegressionTrainerTest.java | 2 +-
.../ignite/ml/tree/DecisionTreeTestSuite.java | 6 +-
.../RandomForestClassifierTrainerTest.java | 51 +-
.../RandomForestRegressionTrainerTest.java | 48 +-
.../ml/tree/randomforest/RandomForestTest.java | 78 +
.../randomforest/RandomForestTreeTestSuite.java | 39 +
.../ml/tree/randomforest/data/TreeNodeTest.java | 79 +
.../data/impurity/GiniFeatureHistogramTest.java | 254 ++
.../data/impurity/ImpurityHistogramTest.java | 69 +
.../data/impurity/MSEHistogramTest.java | 135 +
...ormalDistributionStatisticsComputerTest.java | 130 +
.../osgi-karaf/src/main/resources/features.xml | 4 +-
.../include/ignite/binary/binary_raw_reader.h | 22 +-
.../ignite/impl/binary/binary_reader_impl.h | 61 +-
.../ignite/impl/binary/binary_type_impl.h | 20 +-
.../ignite/impl/binary/binary_writer_impl.h | 48 +-
.../src/impl/binary/binary_reader_impl.cpp | 46 +-
.../src/impl/binary/binary_writer_impl.cpp | 58 +-
.../cpp/common/include/ignite/common/utils.h | 2 +-
.../cpp/core/include/ignite/cache/cache.h | 46 +-
.../include/ignite/cache/query/query_cursor.h | 15 +-
.../cpp/core/include/ignite/impl/operations.h | 150 +-
.../src/impl/cluster/cluster_group_impl.cpp | 2 +-
modules/platforms/cpp/odbc-test/Makefile.am | 5 +-
.../cpp/odbc-test/config/queries-default.xml | 6 +
.../odbc-test/config/queries-transaction-32.xml | 50 +
.../odbc-test/config/queries-transaction.xml | 32 +
.../odbc-test/include/sql_test_suite_fixture.h | 14 +-
.../cpp/odbc-test/project/vs/odbc-test.vcxproj | 5 +
.../project/vs/odbc-test.vcxproj.filters | 15 +
.../src/application_data_buffer_test.cpp | 4 +-
.../cpp/odbc-test/src/connection_test.cpp | 136 +
.../cpp/odbc-test/src/meta_queries_test.cpp | 17 +
.../cpp/odbc-test/src/odbc_test_suite.cpp | 2 +-
.../cpp/odbc-test/src/queries_test.cpp | 10 +-
.../platforms/cpp/odbc-test/src/row_test.cpp | 2 +-
.../src/sql_aggregate_functions_test.cpp | 16 +-
.../src/sql_date_time_functions_test.cpp | 38 +-
.../src/sql_esc_convert_function_test.cpp | 6 +-
.../cpp/odbc-test/src/sql_get_info_test.cpp | 8 +-
.../src/sql_numeric_functions_test.cpp | 4 +-
.../cpp/odbc-test/src/sql_operators_test.cpp | 10 +-
.../odbc-test/src/sql_string_functions_test.cpp | 20 +-
.../odbc-test/src/sql_system_functions_test.cpp | 2 +-
.../odbc-test/src/sql_test_suite_fixture.cpp | 24 +-
.../cpp/odbc-test/src/sql_types_test.cpp | 6 +-
.../src/sql_value_expressions_test.cpp | 6 +-
.../cpp/odbc-test/src/transaction_test.cpp | 738 ++++
modules/platforms/cpp/odbc/Makefile.am | 1 +
.../include/ignite/odbc/config/configuration.h | 32 +
.../odbc/config/connection_string_parser.h | 3 +
.../cpp/odbc/include/ignite/odbc/connection.h | 21 +-
.../ignite/odbc/diagnostic/diagnosable.h | 7 +
.../odbc/diagnostic/diagnosable_adapter.h | 11 +-
.../cpp/odbc/include/ignite/odbc/environment.h | 15 +
.../cpp/odbc/include/ignite/odbc/message.h | 13 +-
.../odbc/include/ignite/odbc/meta/column_meta.h | 47 +-
.../odbc/include/ignite/odbc/nested_tx_mode.h | 81 +
.../odbc/include/ignite/odbc/protocol_version.h | 7 +-
.../odbc/system/ui/dsn_configuration_window.h | 8 +
.../src/system/ui/dsn_configuration_window.cpp | 56 +-
.../platforms/cpp/odbc/project/vs/odbc.vcxproj | 2 +
.../cpp/odbc/project/vs/odbc.vcxproj.filters | 6 +
.../odbc/src/app/application_data_buffer.cpp | 16 +-
.../cpp/odbc/src/config/configuration.cpp | 29 +-
.../cpp/odbc/src/config/connection_info.cpp | 8 +-
.../src/config/connection_string_parser.cpp | 19 +
modules/platforms/cpp/odbc/src/connection.cpp | 99 +-
.../odbc/src/diagnostic/diagnosable_adapter.cpp | 5 +
modules/platforms/cpp/odbc/src/dsn_config.cpp | 5 +
modules/platforms/cpp/odbc/src/environment.cpp | 53 +-
modules/platforms/cpp/odbc/src/message.cpp | 28 +-
.../platforms/cpp/odbc/src/meta/column_meta.cpp | 68 +-
.../platforms/cpp/odbc/src/nested_tx_mode.cpp | 80 +
modules/platforms/cpp/odbc/src/odbc.cpp | 2 +
.../platforms/cpp/odbc/src/protocol_version.cpp | 4 +-
.../cpp/odbc/src/query/batch_query.cpp | 4 +-
.../platforms/cpp/odbc/src/query/data_query.cpp | 2 +-
.../include/ignite/impl/thin/readable.h | 2 +-
.../ExpiryCacheHolderTest.cs | 5 -
.../IgniteSessionStateStoreProviderTest.cs | 7 +-
.../Apache.Ignite.Core.Tests.csproj | 4 +
.../DataStorageConfigurationParityTest.cs | 3 +-
.../ApiParity/IgniteConfigurationParityTest.cs | 5 +-
.../QueryEntityConfigurationParityTest.cs | 3 +-
.../Cache/Query/Linq/CacheLinqTest.Base.cs | 20 +-
.../Query/Linq/CacheLinqTest.Introspection.cs | 22 +-
.../Cache/ClientCacheConfigurationTest.cs | 5 +-
.../Client/ClientConnectionTest.cs | 5 +-
.../Compute/ComputeApiTest.cs | 3 +-
.../Config/full-config.xml | 4 +
.../Apache.Ignite.Core.Tests/EventsTest.cs | 4 +-
.../Examples/PathUtil.cs | 2 +-
.../IgniteConfigurationSerializerTest.cs | 6 +
.../IgniteConfigurationTest.cs | 10 +-
.../IgniteStartStopTest.cs | 23 +-
.../Impl/Compute/ComputeImplTest.cs | 65 +
.../Apache.Ignite.Core.Tests/MessagingTest.cs | 13 +-
.../Apache.Ignite.Core.Tests/ReconnectTest.cs | 3 +-
.../Services/ServicesTest.cs | 3 +-
.../Apache.Ignite.Core.Tests/packages.config | 1 +
.../Cache/Configuration/CacheConfiguration.cs | 27 +-
.../Cache/Configuration/QueryEntity.cs | 12 +-
.../Cache/Configuration/QueryField.cs | 28 +-
.../Configuration/QuerySqlFieldAttribute.cs | 2 +
.../Client/Cache/CacheClientConfiguration.cs | 9 +-
.../Apache.Ignite.Core/Compute/ICompute.cs | 6 +
.../Configuration/DataStorageConfiguration.cs | 14 +
.../Apache.Ignite.Core/IgniteConfiguration.cs | 114 +-
.../IgniteConfigurationSection.xsd | 15 +
.../dotnet/Apache.Ignite.Core/Ignition.cs | 3 +-
.../Impl/Binary/BinaryWriterExtensions.cs | 31 +-
.../Impl/Binary/IBinaryRawWriteAware.cs | 25 +-
.../Apache.Ignite.Core/Impl/Cache/CacheImpl.cs | 3 +-
.../Impl/Client/Cache/CacheClient.cs | 3 +-
.../Cache/ClientCacheConfigurationSerializer.cs | 9 +-
.../Impl/Client/ClientSocket.cs | 22 +-
.../Impl/Client/IgniteClient.cs | 11 +-
.../Apache.Ignite.Core/Impl/Compute/Compute.cs | 8 +
.../Impl/Compute/ComputeImpl.cs | 11 +
.../dotnet/Apache.Ignite.Core/Impl/Ignite.cs | 8 +-
.../Properties/AssemblyInfo.cs | 1 +
.../Apache.Ignite.Linq/Impl/ExpressionWalker.cs | 58 +-
modules/platforms/nodejs/lib/BinaryObject.js | 39 +-
modules/platforms/nodejs/lib/CacheClient.js | 48 +-
.../platforms/nodejs/lib/CacheConfiguration.js | 185 +-
modules/platforms/nodejs/lib/Cursor.js | 22 +-
modules/platforms/nodejs/lib/EnumItem.js | 13 +-
modules/platforms/nodejs/lib/IgniteClient.js | 26 +-
modules/platforms/nodejs/lib/Query.js | 38 +-
.../nodejs/lib/internal/BinaryCommunicator.js | 409 ++
.../nodejs/lib/internal/BinaryReader.js | 197 -
.../platforms/nodejs/lib/internal/BinaryType.js | 33 +-
.../nodejs/lib/internal/BinaryTypeStorage.js | 43 +-
.../nodejs/lib/internal/BinaryWriter.js | 210 -
.../nodejs/lib/internal/ClientSocket.js | 56 +-
.../nodejs/lib/internal/MessageBuffer.js | 6 +
.../nodejs/spec/examples/AuthExample.spec.js | 5 +
.../nodejs/spec/examples/Examples.spec.js | 5 +
modules/rest-http/pom.xml | 22 +
.../http/jetty/GridJettyRestHandler.java | 8 +-
.../http/jetty/GridJettyRestProtocol.java | 12 +-
.../protocols/http/jetty/GridRestSuite.java | 35 +
.../http/jetty/RestSetupSimpleTest.java | 76 +
.../spark/sql/ignite/IgniteSparkSession.scala | 31 +-
.../ignite/spark/IgniteDataFrameSuite.scala | 4 +-
.../sql/ignite/IgniteSparkSessionSpec.scala | 79 +
.../ignite/cache/spring/SpringCacheManager.java | 35 +-
.../spring/SpringTransactionManager.java | 30 +-
...ridSpringCacheManagerSpringBeanSelfTest.java | 14 +-
...ingTransactionManagerSpringBeanSelfTest.java | 6 +
.../util/nodestart/StartNodeCallableImpl.java | 12 +-
.../cluster/TensorFlowClusterMaintainer.java | 142 +-
.../cluster/TensorFlowClusterManager.java | 36 +
.../tfrunning/TensorFlowServerManager.java | 25 +-
.../TensorFlowServerScriptFormatter.java | 3 +
.../cluster/util/ClusterPortManager.java | 16 +-
.../cluster/util/TensorFlowChiefRunner.java | 2 +-
.../util/TensorFlowProcessBuilderSupplier.java | 72 +
.../util/TensorFlowUserScriptRunner.java | 3 +-
.../core/pythonrunning/PythonProcess.java | 68 -
.../PythonProcessBuilderSupplier.java | 93 -
.../pythonrunning/PythonProcessManager.java | 56 -
.../core/pythonrunning/package-info.java | 25 -
.../core/util/AsyncNativeProcessRunner.java | 50 +-
.../core/util/PythonProcessBuilderSupplier.java | 93 +
.../submitter/command/StartCommand.java | 3 +-
.../http/GridHttpDeploymentSelfTest.java | 12 +-
.../web-console/backend/app/agentsHandler.js | 19 +-
.../web-console/backend/app/browsersHandler.js | 12 +-
.../e2e/testcafe/components/FormField.js | 6 +-
.../e2e/testcafe/components/modalInput.js | 8 +-
.../page-models/PageConfigurationBasic.js | 2 +-
modules/web-console/frontend/.babelrc | 16 +-
modules/web-console/frontend/.eslintrc | 2 +-
modules/web-console/frontend/app/app.js | 4 +
.../frontend/app/browserUpdate/index.js | 8 +-
.../components/activities-user-dialog/index.js | 26 +-
.../components/form-field-size/controller.js | 140 +
.../components/form-field-size/index.js | 40 +
.../components/form-field-size/template.pug | 77 +
.../frontend/app/components/form-field/index.js | 3 +
.../components/grid-column-selector/style.scss | 1 +
.../app/components/grid-item-selected/index.js | 1 +
.../components/grid-item-selected/style.scss | 23 +
.../input-dialog/input-dialog.tpl.pug | 13 +-
.../list-editable-cols/cols.style.scss | 8 +-
.../list-editable-cols/cols.template.pug | 2 +-
.../app/components/list-editable/style.scss | 6 +-
.../list-of-registered-users/style.scss | 14 +-
.../list-of-registered-users/template.tpl.pug | 55 +-
.../cache-edit-form/templates/general.pug | 4 +-
.../cache-edit-form/templates/memory.pug | 10 +-
.../cache-edit-form/templates/node-filter.pug | 4 +-
.../cache-edit-form/templates/query.pug | 2 +-
.../cache-edit-form/templates/store.pug | 85 +-
.../cluster-edit-form/templates/attributes.pug | 20 +-
.../cluster-edit-form/templates/binary.pug | 68 +-
.../templates/cache-key-cfg.pug | 62 +-
.../cluster-edit-form/templates/checkpoint.pug | 84 +-
.../templates/collision/job-stealing.pug | 19 +-
.../templates/communication.pug | 10 +-
.../templates/data-storage.pug | 64 +-
.../cluster-edit-form/templates/deployment.pug | 11 +-
.../cluster-edit-form/templates/discovery.pug | 4 +-
.../cluster-edit-form/templates/failover.pug | 77 +-
.../cluster-edit-form/templates/general.pug | 2 +-
.../templates/general/discovery/jdbc.pug | 8 +-
.../templates/general/discovery/multicast.pug | 42 +-
.../templates/general/discovery/vm.pug | 53 +-
.../templates/load-balancing.pug | 155 +-
.../cluster-edit-form/templates/memory.pug | 195 +-
.../cluster-edit-form/templates/odbc.pug | 4 +-
.../cluster-edit-form/templates/service.pug | 90 +-
.../cluster-edit-form/templates/ssl.pug | 43 +-
.../cluster-edit-form/templates/swap.pug | 4 +-
.../cluster-edit-form/templates/thread.pug | 101 +-
.../igfs-edit-form/templates/general.pug | 6 +-
.../igfs-edit-form/templates/misc.pug | 72 +-
.../igfs-edit-form/templates/secondary.pug | 6 +-
.../model-edit-form/templates/general.pug | 2 +-
.../model-edit-form/templates/query.pug | 348 +-
.../model-edit-form/templates/store.pug | 46 +-
.../components/page-configure-basic/style.scss | 29 +-
.../page-configure-basic/template.pug | 72 +-
.../components/modal-import-models/component.js | 2 +-
.../tables-action-cell/style.scss | 1 -
.../tables-action-cell/template.pug | 34 +-
.../modal-import-models/template.tpl.pug | 135 +-
.../modal-preview-project/template.pug | 3 +-
.../components/pc-form-field-size/component.js | 41 -
.../components/pc-form-field-size/controller.js | 135 -
.../components/pc-form-field-size/index.js | 23 -
.../components/pc-form-field-size/style.scss | 52 -
.../components/pc-form-field-size/template.pug | 61 -
.../components/pc-items-table/style.scss | 3 -
.../components/pc-items-table/template.pug | 36 +-
.../app/components/page-configure/index.js | 2 -
.../services/ConfigurationResource.js | 4 +-
.../services/ConfigurationResource.spec.js | 78 +
.../app/components/page-configure/states.js | 3 +-
.../app/components/page-configure/style.scss | 27 +-
.../app/components/page-configure/template.pug | 2 +-
.../components/queries-notebook/controller.js | 16 +-
.../components/queries-notebook/style.scss | 20 +
.../queries-notebook/template.tpl.pug | 262 +-
.../queries-notebooks-list/template.tpl.pug | 29 +-
.../app/components/page-queries/style.scss | 4 +
.../app/components/ui-grid/component.js | 38 +
.../app/components/ui-grid/controller.js | 139 +
.../frontend/app/components/ui-grid/index.js | 23 +
.../frontend/app/components/ui-grid/style.scss | 111 +
.../app/components/ui-grid/template.pug | 34 +
.../components/user-notifications/style.scss | 39 -
.../user-notifications/template.tpl.pug | 26 +-
.../app/directives/auto-focus.directive.js | 2 +-
.../app/directives/bs-affix-update.directive.js | 2 +-
.../frontend/app/directives/btn-ignite-link.js | 22 +-
.../directives/centered/centered.directive.js | 2 +-
.../directives/copy-to-clipboard.directive.js | 2 +-
.../hide-on-state-change.directive.js | 2 +-
.../information/information.directive.js | 2 +-
.../app/directives/on-click-focus.directive.js | 2 +-
.../directives/on-enter-focus-move.directive.js | 2 +-
.../app/directives/on-enter.directive.js | 2 +-
.../app/directives/on-escape.directive.js | 2 +-
.../directives/restore-input-focus.directive.js | 2 +-
.../directives/retain-selection.directive.js | 2 +-
.../ui-ace-docker/ui-ace-docker.directive.js | 2 +-
.../ui-ace-java/ui-ace-java.directive.js | 4 +-
.../ui-ace-pojos/ui-ace-pojos.directive.js | 2 +-
.../ui-ace-pom/ui-ace-pom.directive.js | 2 +-
.../ui-ace-sharp/ui-ace-sharp.directive.js | 4 +-
.../ui-ace-spring/ui-ace-spring.directive.js | 4 +-
.../app/directives/ui-ace-tabs.directive.js | 4 +-
.../frontend/app/helpers/jade/form.pug | 26 -
.../helpers/jade/form/form-field-checkbox.pug | 44 -
.../helpers/jade/form/form-field-datalist.pug | 52 -
.../app/helpers/jade/form/form-field-down.pug | 18 -
.../helpers/jade/form/form-field-dropdown.pug | 60 -
.../helpers/jade/form/form-field-feedback.pug | 18 -
.../app/helpers/jade/form/form-field-label.pug | 25 -
.../app/helpers/jade/form/form-field-number.pug | 59 -
.../helpers/jade/form/form-field-password.pug | 47 -
.../app/helpers/jade/form/form-field-text.pug | 53 -
.../app/helpers/jade/form/form-field-up.pug | 18 -
.../frontend/app/helpers/jade/mixins.pug | 598 ++-
.../frontend/app/modules/ace.module.js | 2 +-
.../app/modules/agent/AgentManager.service.js | 2 +
.../app/modules/agent/AgentModal.service.js | 2 +-
.../app/modules/branding/features.directive.js | 2 +-
.../app/modules/branding/footer.directive.js | 2 +-
.../modules/branding/header-logo.directive.js | 2 +-
.../configuration/configuration.module.js | 6 +-
.../generator/PlatformGenerator.js | 4 +-
.../frontend/app/modules/demo/Demo.module.js | 6 +-
.../app/modules/dialog/dialog.factory.js | 2 +-
.../app/modules/form/field/down.directive.js | 45 -
.../app/modules/form/field/feedback.scss | 37 -
.../frontend/app/modules/form/field/field.scss | 43 -
.../field/form-control-feedback.directive.js | 40 -
.../app/modules/form/field/input/text.scss | 41 -
.../app/modules/form/field/label.directive.js | 47 -
.../app/modules/form/field/tooltip.directive.js | 49 -
.../app/modules/form/field/up.directive.js | 43 -
.../frontend/app/modules/form/form.module.js | 28 -
.../app/modules/form/group/add.directive.js | 38 -
.../app/modules/form/group/tooltip.directive.js | 38 -
.../app/modules/form/panel/chevron.directive.js | 2 +-
.../modules/form/services/FormGUID.service.js | 2 +-
.../app/modules/loading/loading.service.js | 2 +-
.../frontend/app/modules/socket.module.js | 2 +-
.../frontend/app/modules/states/logout.state.js | 2 +-
.../frontend/app/modules/user/user.module.js | 2 +-
.../frontend/app/primitives/btn/index.scss | 5 +-
.../frontend/app/primitives/checkbox/index.scss | 20 -
.../app/primitives/datepicker/index.pug | 29 +-
.../app/primitives/datepicker/index.scss | 55 -
.../frontend/app/primitives/file/index.pug | 37 -
.../frontend/app/primitives/file/index.scss | 60 -
.../app/primitives/form-field/checkbox.pug | 4 +-
.../app/primitives/form-field/dropdown.pug | 17 +-
.../app/primitives/form-field/email.pug | 4 +-
.../app/primitives/form-field/error.pug | 1 +
.../app/primitives/form-field/index.pug | 2 +
.../app/primitives/form-field/index.scss | 286 +-
.../app/primitives/form-field/input.pug | 7 +-
.../app/primitives/form-field/label.pug | 27 +-
.../app/primitives/form-field/number.pug | 19 +-
.../app/primitives/form-field/password.pug | 4 +-
.../app/primitives/form-field/phone.pug | 2 +-
.../app/primitives/form-field/radio.pug | 32 +
.../frontend/app/primitives/form-field/text.pug | 9 +-
.../app/primitives/form-field/tooltip.pug | 12 +-
.../app/primitives/form-field/typeahead.pug | 54 +
.../frontend/app/primitives/index.js | 1 -
.../frontend/app/primitives/modal/index.scss | 27 +-
.../frontend/app/primitives/panel/index.scss | 9 +
.../frontend/app/primitives/radio/index.pug | 37 -
.../frontend/app/primitives/radio/index.scss | 78 -
.../app/primitives/timepicker/index.pug | 31 +-
.../app/primitives/timepicker/index.scss | 130 +-
.../frontend/app/primitives/tooltip/index.pug | 27 -
.../services/AngularStrapSelect.decorator.js | 2 +-
.../services/AngularStrapTooltip.decorator.js | 2 +-
.../frontend/app/services/Confirm.service.js | 4 +-
.../app/services/CopyToClipboard.service.js | 2 +-
.../frontend/app/services/Focus.service.js | 2 +-
.../frontend/app/services/FormUtils.service.js | 2 +-
.../app/services/LegacyTable.service.js | 2 +-
.../app/services/LegacyUtils.service.js | 2 +-
.../frontend/app/services/Messages.service.js | 2 +-
.../app/services/ModelNormalizer.service.js | 2 +-
.../app/services/UnsavedChangesGuard.service.js | 2 +-
modules/web-console/frontend/app/style.scss | 22 +
modules/web-console/frontend/app/vendor.js | 1 -
modules/web-console/frontend/package.json | 41 +-
.../frontend/public/images/icons/index.js | 56 +-
.../frontend/public/stylesheets/style.scss | 77 +-
.../frontend/test/karma.conf.babel.js | 113 -
modules/web-console/frontend/test/karma.conf.js | 97 +-
.../frontend/views/templates/confirm.tpl.pug | 9 +-
.../frontend/webpack/webpack.common.js | 71 +-
.../frontend/webpack/webpack.dev.babel.js | 92 -
.../web-console/frontend/webpack/webpack.dev.js | 92 +
.../frontend/webpack/webpack.prod.babel.js | 61 -
.../frontend/webpack/webpack.prod.js | 61 +
.../frontend/webpack/webpack.test.js | 7 +-
.../internal/websession/WebSessionSelfTest.java | 39 +-
.../config/mvcc/benchmark-mvcc-messages.sh | 98 +
.../config/mvcc/benchmark-mvcc-processor.sh | 95 +
.../mvcc/benchmark-mvcc-updates-contention.sh | 96 +
.../mvcc/benchmark-thin-native.properties | 124 +
...benchmark-jdbc-thin-inmemory-mvcc.properties | 105 +
.../yardstick/IgniteBenchmarkArguments.java | 38 +
.../org/apache/ignite/yardstick/IgniteNode.java | 2 +
.../yardstick/IgniteThinAbstractBenchmark.java | 164 +
.../ignite/yardstick/IgniteThinClient.java | 146 +
.../cache/IgnitePutAllSimpleBenchmark.java | 44 +
.../yardstick/jdbc/AbstractJdbcBenchmark.java | 2 +-
.../yardstick/jdbc/AbstractNativeBenchmark.java | 2 +-
.../apache/ignite/yardstick/jdbc/JdbcUtils.java | 11 +-
.../jdbc/NativeJavaApiPutRemoveBenchmark.java | 52 +
.../mvcc/AbstractDistributedMvccBenchmark.java | 101 +
.../jdbc/mvcc/MvccProcessorBenchmark.java | 61 +
.../mvcc/MvccUpdateContentionBenchmark.java | 77 +
.../thin/cache/IgniteThinBenchmarkUtils.java | 95 +
.../cache/IgniteThinCacheAbstractBenchmark.java | 45 +
.../thin/cache/IgniteThinGetBenchmark.java | 74 +
.../thin/cache/IgniteThinGetTxBenchmark.java | 30 +
.../thin/cache/IgniteThinPutAllBenchmark.java | 44 +
.../thin/cache/IgniteThinPutAllTxBenchmark.java | 30 +
.../thin/cache/IgniteThinPutBenchmark.java | 41 +
.../thin/cache/IgniteThinPutGetBenchmark.java | 46 +
.../thin/cache/IgniteThinPutTxBenchmark.java | 30 +
.../upload/AbstractNativeBenchmark.java | 14 +-
.../upload/AbstractUploadBenchmark.java | 9 +-
.../yardstick/upload/model/QueryFactory.java | 30 +-
modules/yarn/pom.xml | 20 +-
.../ZkDistributedCollectDataFuture.java | 19 +-
.../discovery/zk/internal/ZookeeperClient.java | 40 +-
.../zk/internal/ZookeeperClusterNode.java | 5 -
.../zk/internal/ZookeeperDiscoveryImpl.java | 39 +-
.../tcp/ipfinder/zk/ZookeeperIpFinderTest.java | 9 +-
.../zk/curator/FixedTestingQuorumPeerMain.java | 87 +
.../tcp/ipfinder/zk/curator/TestingCluster.java | 237 ++
.../zk/curator/TestingZooKeeperServer.java | 169 +
.../ZookeeperDiscoverySpiAbstractTestSuite.java | 2 +-
.../zk/ZookeeperDiscoverySpiTestSuite2.java | 2 +-
.../zk/internal/ZookeeperClientTest.java | 15 +-
.../zk/internal/ZookeeperDiscoverySpiTest.java | 4 +-
parent/pom.xml | 24 +-
1467 files changed, 98755 insertions(+), 23068 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a7ee714e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 8a51bf5,714d4a0..4806733
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@@ -161,12 -182,8 +183,12 @@@ public abstract class GridCacheMapEntr
/** */
@GridToStringExclude
- private final ReentrantLock lock = new ReentrantLock();
-
+ private final ReentrantLock lock = new ReentrantLock();
+
+ /** Read Lock for continuous query listener */
+ @GridToStringExclude
+ private final Lock listenerLock;
+
/**
* Flags:
* <ul>
@@@ -958,7 -1369,8 +1375,9 @@@
ensureFreeSpace();
+ GridLongList mvccWaitTxs = null;
+
+ lockListenerReadLock();
lockEntry();
try {
@@@ -1177,7 -1600,8 +1608,9 @@@
boolean marked = false;
+ GridLongList mvccWaitTxs = null;
+
+ lockListenerReadLock();
lockEntry();
try {
@@@ -4344,27 -4851,7 +4867,27 @@@
@Override public void unlockEntry() {
lock.unlock();
}
-
+
+ /**
+ * This method would obtain read lock for continuous query listener setup. This
+ * is to prevent race condition between entry update and continuous query setup.
+ * You should make sure you obtain this read lock first before locking the entry
+ * in order to ensure that the entry update is completed and existing continuous
+ * query notified before the next cache listener update
+ */
+ private void lockListenerReadLock() {
+ listenerLock.lock();
- }
-
++ }
++
+ /**
+ * unlock the listener read lock
- *
++ *
+ * @see #lockListenerReadLock()
+ */
+ private void unlockListenerReadLock() {
+ listenerLock.unlock();
+ }
-
++
/** {@inheritDoc} */
@Override public boolean lockedByCurrentThread() {
return lock.isHeldByCurrentThread();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a7ee714e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
----------------------------------------------------------------------
[36/50] [abbrv] ignite git commit: IGNITE-9412: [ML] GDB convergence
by error support.
Posted by ag...@apache.org.
IGNITE-9412: [ML] GDB convergence by error support.
this closes #4670
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6225c56e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6225c56e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6225c56e
Branch: refs/heads/ignite-5960
Commit: 6225c56ea70e0af26eee51c7e5e7e53af93386ca
Parents: ed6bf5a
Author: Alexey Platonov <ap...@gmail.com>
Authored: Thu Sep 6 12:08:36 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Thu Sep 6 12:08:36 2018 +0300
----------------------------------------------------------------------
.../GDBOnTreesClassificationTrainerExample.java | 10 +-
.../GDBOnTreesRegressionTrainerExample.java | 4 +-
.../boosting/GDBBinaryClassifierTrainer.java | 57 +++---
.../boosting/GDBLearningStrategy.java | 115 ++++++++++--
.../boosting/GDBRegressionTrainer.java | 8 +-
.../ml/composition/boosting/GDBTrainer.java | 122 ++++++++----
.../LossGradientPerPredictionFunctions.java | 33 ----
.../convergence/ConvergenceChecker.java | 140 ++++++++++++++
.../convergence/ConvergenceCheckerFactory.java | 58 ++++++
.../mean/MeanAbsValueConvergenceChecker.java | 116 ++++++++++++
.../MeanAbsValueConvergenceCheckerFactory.java | 47 +++++
.../boosting/convergence/mean/package-info.java | 22 +++
.../MedianOfMedianConvergenceChecker.java | 126 +++++++++++++
...MedianOfMedianConvergenceCheckerFactory.java | 47 +++++
.../convergence/median/package-info.java | 22 +++
.../boosting/convergence/package-info.java | 24 +++
.../simple/ConvergenceCheckerStub.java | 79 ++++++++
.../simple/ConvergenceCheckerStubFactory.java | 48 +++++
.../convergence/simple/package-info.java | 24 +++
.../ml/composition/boosting/loss/LogLoss.java | 36 ++++
.../ml/composition/boosting/loss/Loss.java | 45 +++++
.../composition/boosting/loss/SquaredError.java | 36 ++++
.../composition/boosting/loss/package-info.java | 22 +++
.../WeightedPredictionsAggregator.java | 10 +
.../FeatureMatrixWithLabelsOnHeapData.java | 57 ++++++
...eatureMatrixWithLabelsOnHeapDataBuilder.java | 76 ++++++++
.../boosting/GDBOnTreesLearningStrategy.java | 24 ++-
.../ignite/ml/tree/data/DecisionTreeData.java | 36 ++--
.../ml/composition/boosting/GDBTrainerTest.java | 81 +++++++-
.../convergence/ConvergenceCheckerTest.java | 82 ++++++++
.../MeanAbsValueConvergenceCheckerTest.java | 73 ++++++++
.../MedianOfMedianConvergenceCheckerTest.java | 57 ++++++
.../ml/environment/LearningEnvironmentTest.java | 187 +++----------------
.../ignite/ml/knn/ANNClassificationTest.java | 12 +-
34 files changed, 1608 insertions(+), 328 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java
index 075eab2..e092e5c 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java
@@ -22,9 +22,8 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.Model;
import org.apache.ignite.ml.composition.ModelsComposition;
-import org.apache.ignite.ml.math.primitives.vector.Vector;
+import org.apache.ignite.ml.composition.boosting.convergence.mean.MeanAbsValueConvergenceCheckerFactory;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.trainers.DatasetTrainer;
import org.apache.ignite.ml.tree.boosting.GDBBinaryClassifierOnTreesTrainer;
@@ -59,10 +58,11 @@ public class GDBOnTreesClassificationTrainerExample {
IgniteCache<Integer, double[]> trainingSet = fillTrainingData(ignite, trainingSetCfg);
// Create regression trainer.
- DatasetTrainer<ModelsComposition, Double> trainer = new GDBBinaryClassifierOnTreesTrainer(1.0, 300, 2, 0.);
+ DatasetTrainer<ModelsComposition, Double> trainer = new GDBBinaryClassifierOnTreesTrainer(1.0, 300, 2, 0.)
+ .withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.1));
// Train decision tree model.
- Model<Vector, Double> mdl = trainer.fit(
+ ModelsComposition mdl = trainer.fit(
ignite,
trainingSet,
(k, v) -> VectorUtils.of(v[0]),
@@ -81,6 +81,8 @@ public class GDBOnTreesClassificationTrainerExample {
}
System.out.println(">>> ---------------------------------");
+ System.out.println(">>> Count of trees = " + mdl.getModels().size());
+ System.out.println(">>> ---------------------------------");
System.out.println(">>> GDB classification trainer example completed.");
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java
index b2b08d0..3662973 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java
@@ -24,6 +24,7 @@ import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.ml.Model;
import org.apache.ignite.ml.composition.ModelsComposition;
+import org.apache.ignite.ml.composition.boosting.convergence.mean.MeanAbsValueConvergenceCheckerFactory;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.trainers.DatasetTrainer;
@@ -59,7 +60,8 @@ public class GDBOnTreesRegressionTrainerExample {
IgniteCache<Integer, double[]> trainingSet = fillTrainingData(ignite, trainingSetCfg);
// Create regression trainer.
- DatasetTrainer<ModelsComposition, Double> trainer = new GDBRegressionOnTreesTrainer(1.0, 2000, 1, 0.);
+ DatasetTrainer<ModelsComposition, Double> trainer = new GDBRegressionOnTreesTrainer(1.0, 2000, 1, 0.)
+ .withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.001));
// Train decision tree model.
Model<Vector, Double> mdl = trainer.fit(
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBBinaryClassifierTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBBinaryClassifierTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBBinaryClassifierTrainer.java
index 3701557..f6ddfed 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBBinaryClassifierTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBBinaryClassifierTrainer.java
@@ -19,24 +19,23 @@ package org.apache.ignite.ml.composition.boosting;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.ml.composition.boosting.loss.LogLoss;
+import org.apache.ignite.ml.composition.boosting.loss.Loss;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.dataset.primitive.builder.context.EmptyContextBuilder;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteTriFunction;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.structures.LabeledVector;
import org.apache.ignite.ml.structures.LabeledVectorSet;
import org.apache.ignite.ml.structures.partition.LabeledDatasetPartitionDataBuilderOnHeap;
/**
- * Trainer for binary classifier using Gradient Boosting.
- * As preparing stage this algorithm learn labels in dataset and create mapping dataset labels to 0 and 1.
- * This algorithm uses gradient of Logarithmic Loss metric [LogLoss] by default in each step of learning.
+ * Trainer for binary classifier using Gradient Boosting. As preparing stage this algorithm learn labels in dataset and
+ * create mapping dataset labels to 0 and 1. This algorithm uses gradient of Logarithmic Loss metric [LogLoss] by
+ * default in each step of learning.
*/
public abstract class GDBBinaryClassifierTrainer extends GDBTrainer {
/** External representation of first class. */
@@ -51,9 +50,7 @@ public abstract class GDBBinaryClassifierTrainer extends GDBTrainer {
* @param cntOfIterations Count of learning iterations.
*/
public GDBBinaryClassifierTrainer(double gradStepSize, Integer cntOfIterations) {
- super(gradStepSize,
- cntOfIterations,
- LossGradientPerPredictionFunctions.LOG_LOSS);
+ super(gradStepSize, cntOfIterations, new LogLoss());
}
/**
@@ -61,35 +58,37 @@ public abstract class GDBBinaryClassifierTrainer extends GDBTrainer {
*
* @param gradStepSize Grad step size.
* @param cntOfIterations Count of learning iterations.
- * @param lossGradient Gradient of loss function. First argument is sample size, second argument is valid answer, third argument is current model prediction.
+ * @param loss Loss function.
*/
- public GDBBinaryClassifierTrainer(double gradStepSize,
- Integer cntOfIterations,
- IgniteTriFunction<Long, Double, Double, Double> lossGradient) {
-
- super(gradStepSize, cntOfIterations, lossGradient);
+ public GDBBinaryClassifierTrainer(double gradStepSize, Integer cntOfIterations, Loss loss) {
+ super(gradStepSize, cntOfIterations, loss);
}
/** {@inheritDoc} */
- @Override protected <V, K> void learnLabels(DatasetBuilder<K, V> builder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ @Override protected <V, K> boolean learnLabels(DatasetBuilder<K, V> builder,
+ IgniteBiFunction<K, V, Vector> featureExtractor,
IgniteBiFunction<K, V, Double> lExtractor) {
- List<Double> uniqLabels = new ArrayList<Double>(
- builder.build(new EmptyContextBuilder<>(), new LabeledDatasetPartitionDataBuilderOnHeap<>(featureExtractor, lExtractor))
- .compute((IgniteFunction<LabeledVectorSet<Double,LabeledVector>, Set<Double>>) x ->
+ Set<Double> uniqLabels = builder.build(new EmptyContextBuilder<>(), new LabeledDatasetPartitionDataBuilderOnHeap<>(featureExtractor, lExtractor))
+ .compute((IgniteFunction<LabeledVectorSet<Double, LabeledVector>, Set<Double>>)x ->
Arrays.stream(x.labels()).boxed().collect(Collectors.toSet()), (a, b) -> {
- if (a == null)
- return b;
- if (b == null)
- return a;
- a.addAll(b);
+ if (a == null)
+ return b;
+ if (b == null)
return a;
- }
- ));
+ a.addAll(b);
+ return a;
+ }
+ );
- A.ensure(uniqLabels.size() == 2, "Binary classifier expects two types of labels in learning dataset");
- externalFirstCls = uniqLabels.get(0);
- externalSecondCls = uniqLabels.get(1);
+ if (uniqLabels != null && uniqLabels.size() == 2) {
+ ArrayList<Double> lblsArray = new ArrayList<>(uniqLabels);
+ externalFirstCls = lblsArray.get(0);
+ externalSecondCls = lblsArray.get(1);
+ return true;
+ } else {
+ return false;
+ }
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBLearningStrategy.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBLearningStrategy.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBLearningStrategy.java
index 375748a..737495e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBLearningStrategy.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBLearningStrategy.java
@@ -22,6 +22,10 @@ import java.util.Arrays;
import java.util.List;
import org.apache.ignite.ml.Model;
import org.apache.ignite.ml.composition.ModelsComposition;
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceChecker;
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceCheckerFactory;
+import org.apache.ignite.ml.composition.boosting.convergence.mean.MeanAbsValueConvergenceCheckerFactory;
+import org.apache.ignite.ml.composition.boosting.loss.Loss;
import org.apache.ignite.ml.composition.predictionsaggregator.WeightedPredictionsAggregator;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.environment.LearningEnvironment;
@@ -29,9 +33,9 @@ import org.apache.ignite.ml.environment.logging.MLLogger;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.math.functions.IgniteTriFunction;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.trainers.DatasetTrainer;
+import org.jetbrains.annotations.NotNull;
/**
* Learning strategy for gradient boosting.
@@ -44,7 +48,7 @@ public class GDBLearningStrategy {
protected int cntOfIterations;
/** Loss of gradient. */
- protected IgniteTriFunction<Long, Double, Double, Double> lossGradient;
+ protected Loss loss;
/** External label to internal mapping. */
protected IgniteFunction<Double, Double> externalLbToInternalMapping;
@@ -61,9 +65,15 @@ public class GDBLearningStrategy {
/** Composition weights. */
protected double[] compositionWeights;
+ /** Check convergence strategy factory. */
+ protected ConvergenceCheckerFactory checkConvergenceStgyFactory = new MeanAbsValueConvergenceCheckerFactory(0.001);
+
+ /** Default gradient step size. */
+ private double defaultGradStepSize;
+
/**
- * Implementation of gradient boosting iterations. At each step of iterations this algorithm
- * build a regression model based on gradient of loss-function for current models composition.
+ * Implementation of gradient boosting iterations. At each step of iterations this algorithm build a regression
+ * model based on gradient of loss-function for current models composition.
*
* @param datasetBuilder Dataset builder.
* @param featureExtractor Feature extractor.
@@ -73,18 +83,43 @@ public class GDBLearningStrategy {
public <K, V> List<Model<Vector, Double>> learnModels(DatasetBuilder<K, V> datasetBuilder,
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
- List<Model<Vector, Double>> models = new ArrayList<>();
+ return update(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /**
+ * Gets state of model in arguments, compare it with training parameters of trainer and if they are fit then
+ * trainer updates model in according to new data and return new model. In other case trains new model.
+ *
+ * @param mdlToUpdate Learned model.
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ * @return Updated models list.
+ */
+ public <K,V> List<Model<Vector, Double>> update(GDBTrainer.GDBModel mdlToUpdate,
+ DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ List<Model<Vector, Double>> models = initLearningState(mdlToUpdate);
+
+ ConvergenceChecker<K, V> convCheck = checkConvergenceStgyFactory.create(sampleSize,
+ externalLbToInternalMapping, loss, datasetBuilder, featureExtractor, lbExtractor);
+
DatasetTrainer<? extends Model<Vector, Double>, Double> trainer = baseMdlTrainerBuilder.get();
for (int i = 0; i < cntOfIterations; i++) {
- double[] weights = Arrays.copyOf(compositionWeights, i);
+ double[] weights = Arrays.copyOf(compositionWeights, models.size());
WeightedPredictionsAggregator aggregator = new WeightedPredictionsAggregator(weights, meanLabelValue);
- Model<Vector, Double> currComposition = new ModelsComposition(models, aggregator);
+ ModelsComposition currComposition = new ModelsComposition(models, aggregator);
+ if (convCheck.isConverged(datasetBuilder, currComposition))
+ break;
IgniteBiFunction<K, V, Double> lbExtractorWrap = (k, v) -> {
Double realAnswer = externalLbToInternalMapping.apply(lbExtractor.apply(k, v));
Double mdlAnswer = currComposition.apply(featureExtractor.apply(k, v));
- return -lossGradient.apply(sampleSize, realAnswer, mdlAnswer);
+ return -loss.gradient(sampleSize, realAnswer, mdlAnswer);
};
long startTs = System.currentTimeMillis();
@@ -97,6 +132,29 @@ public class GDBLearningStrategy {
}
/**
+ * Restores state of already learned model if can and sets learning parameters according to this state.
+ *
+ * @param mdlToUpdate Model to update.
+ * @return list of already learned models.
+ */
+ @NotNull protected List<Model<Vector, Double>> initLearningState(GDBTrainer.GDBModel mdlToUpdate) {
+ List<Model<Vector, Double>> models = new ArrayList<>();
+ if(mdlToUpdate != null) {
+ models.addAll(mdlToUpdate.getModels());
+ WeightedPredictionsAggregator aggregator = (WeightedPredictionsAggregator) mdlToUpdate.getPredictionsAggregator();
+ meanLabelValue = aggregator.getBias();
+ compositionWeights = new double[models.size() + cntOfIterations];
+ for(int i = 0; i < models.size(); i++)
+ compositionWeights[i] = aggregator.getWeights()[i];
+ } else {
+ compositionWeights = new double[cntOfIterations];
+ }
+
+ Arrays.fill(compositionWeights, models.size(), compositionWeights.length, defaultGradStepSize);
+ return models;
+ }
+
+ /**
* Sets learning environment.
*
* @param environment Learning Environment.
@@ -117,12 +175,12 @@ public class GDBLearningStrategy {
}
/**
- * Sets gradient of loss function.
+ * Loss function.
*
- * @param lossGradient Loss gradient.
+ * @param loss Loss function.
*/
- public GDBLearningStrategy withLossGradient(IgniteTriFunction<Long, Double, Double, Double> lossGradient) {
- this.lossGradient = lossGradient;
+ public GDBLearningStrategy withLossGradient(Loss loss) {
+ this.loss = loss;
return this;
}
@@ -141,7 +199,8 @@ public class GDBLearningStrategy {
*
* @param buildBaseMdlTrainer Build base model trainer.
*/
- public GDBLearningStrategy withBaseModelTrainerBuilder(IgniteSupplier<DatasetTrainer<? extends Model<Vector, Double>, Double>> buildBaseMdlTrainer) {
+ public GDBLearningStrategy withBaseModelTrainerBuilder(
+ IgniteSupplier<DatasetTrainer<? extends Model<Vector, Double>, Double>> buildBaseMdlTrainer) {
this.baseMdlTrainerBuilder = buildBaseMdlTrainer;
return this;
}
@@ -175,4 +234,34 @@ public class GDBLearningStrategy {
this.compositionWeights = compositionWeights;
return this;
}
+
+ /**
+ * Sets CheckConvergenceStgyFactory.
+ *
+ * @param factory Factory.
+ */
+ public GDBLearningStrategy withCheckConvergenceStgyFactory(ConvergenceCheckerFactory factory) {
+ this.checkConvergenceStgyFactory = factory;
+ return this;
+ }
+
+ /**
+ * Sets default gradient step size.
+ *
+ * @param defaultGradStepSize Default gradient step size.
+ */
+ public GDBLearningStrategy withDefaultGradStepSize(double defaultGradStepSize) {
+ this.defaultGradStepSize = defaultGradStepSize;
+ return this;
+ }
+
+ /** */
+ public double[] getCompositionWeights() {
+ return compositionWeights;
+ }
+
+ /** */
+ public double getMeanValue() {
+ return meanLabelValue;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBRegressionTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBRegressionTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBRegressionTrainer.java
index 201586e..8c1afd7 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBRegressionTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBRegressionTrainer.java
@@ -17,6 +17,7 @@
package org.apache.ignite.ml.composition.boosting;
+import org.apache.ignite.ml.composition.boosting.loss.SquaredError;
import org.apache.ignite.ml.dataset.DatasetBuilder;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.primitives.vector.Vector;
@@ -33,15 +34,14 @@ public abstract class GDBRegressionTrainer extends GDBTrainer {
* @param cntOfIterations Count of learning iterations.
*/
public GDBRegressionTrainer(double gradStepSize, Integer cntOfIterations) {
- super(gradStepSize,
- cntOfIterations,
- LossGradientPerPredictionFunctions.MSE);
+ super(gradStepSize, cntOfIterations, new SquaredError());
}
/** {@inheritDoc} */
- @Override protected <V, K> void learnLabels(DatasetBuilder<K, V> builder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ @Override protected <V, K> boolean learnLabels(DatasetBuilder<K, V> builder, IgniteBiFunction<K, V, Vector> featureExtractor,
IgniteBiFunction<K, V, Double> lExtractor) {
+ return true;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java
index c7f21dd..85af798 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/GDBTrainer.java
@@ -22,6 +22,9 @@ import java.util.List;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.ml.Model;
import org.apache.ignite.ml.composition.ModelsComposition;
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceCheckerFactory;
+import org.apache.ignite.ml.composition.boosting.convergence.mean.MeanAbsValueConvergenceCheckerFactory;
+import org.apache.ignite.ml.composition.boosting.loss.Loss;
import org.apache.ignite.ml.composition.predictionsaggregator.WeightedPredictionsAggregator;
import org.apache.ignite.ml.dataset.Dataset;
import org.apache.ignite.ml.dataset.DatasetBuilder;
@@ -30,7 +33,7 @@ import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
import org.apache.ignite.ml.environment.logging.MLLogger;
import org.apache.ignite.ml.knn.regression.KNNRegressionTrainer;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteTriFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.regressions.linear.LinearRegressionLSQRTrainer;
import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
@@ -60,24 +63,25 @@ public abstract class GDBTrainer extends DatasetTrainer<ModelsComposition, Doubl
private final int cntOfIterations;
/**
- * Gradient of loss function. First argument is sample size, second argument is valid answer, third argument is
- * current model prediction.
+ * Loss function.
*/
- protected final IgniteTriFunction<Long, Double, Double, Double> lossGradient;
+ protected final Loss loss;
+
+ /** Check convergence strategy factory. */
+ protected ConvergenceCheckerFactory checkConvergenceStgyFactory = new MeanAbsValueConvergenceCheckerFactory(0.001);
/**
* Constructs GDBTrainer instance.
*
* @param gradStepSize Grad step size.
* @param cntOfIterations Count of learning iterations.
- * @param lossGradient Gradient of loss function. First argument is sample size, second argument is valid answer
+ * @param loss Gradient of loss function. First argument is sample size, second argument is valid answer
* third argument is current model prediction.
*/
- public GDBTrainer(double gradStepSize, Integer cntOfIterations,
- IgniteTriFunction<Long, Double, Double, Double> lossGradient) {
+ public GDBTrainer(double gradStepSize, Integer cntOfIterations, Loss loss) {
gradientStep = gradStepSize;
this.cntOfIterations = cntOfIterations;
- this.lossGradient = lossGradient;
+ this.loss = loss;
}
/** {@inheritDoc} */
@@ -85,53 +89,55 @@ public abstract class GDBTrainer extends DatasetTrainer<ModelsComposition, Doubl
IgniteBiFunction<K, V, Vector> featureExtractor,
IgniteBiFunction<K, V, Double> lbExtractor) {
- learnLabels(datasetBuilder, featureExtractor, lbExtractor);
+ return updateModel(null, datasetBuilder, featureExtractor, lbExtractor);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected <K, V> ModelsComposition updateModel(ModelsComposition mdl, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ if (!learnLabels(datasetBuilder, featureExtractor, lbExtractor))
+ return getLastTrainedModelOrThrowEmptyDatasetException(mdl);
+
+ IgniteBiTuple<Double, Long> initAndSampleSize = computeInitialValue(datasetBuilder, featureExtractor, lbExtractor);
+ if(initAndSampleSize == null)
+ return getLastTrainedModelOrThrowEmptyDatasetException(mdl);
- IgniteBiTuple<Double, Long> initAndSampleSize = computeInitialValue(datasetBuilder,
- featureExtractor, lbExtractor);
Double mean = initAndSampleSize.get1();
Long sampleSize = initAndSampleSize.get2();
- double[] compositionWeights = new double[cntOfIterations];
- Arrays.fill(compositionWeights, gradientStep);
- WeightedPredictionsAggregator resAggregator = new WeightedPredictionsAggregator(compositionWeights, mean);
-
long learningStartTs = System.currentTimeMillis();
- List<Model<Vector, Double>> models = getLearningStrategy()
+ GDBLearningStrategy stgy = getLearningStrategy()
.withBaseModelTrainerBuilder(this::buildBaseModelTrainer)
.withExternalLabelToInternal(this::externalLabelToInternal)
.withCntOfIterations(cntOfIterations)
- .withCompositionWeights(compositionWeights)
.withEnvironment(environment)
- .withLossGradient(lossGradient)
+ .withLossGradient(loss)
.withSampleSize(sampleSize)
.withMeanLabelValue(mean)
- .learnModels(datasetBuilder, featureExtractor, lbExtractor);
+ .withDefaultGradStepSize(gradientStep)
+ .withCheckConvergenceStgyFactory(checkConvergenceStgyFactory);
+
+ List<Model<Vector, Double>> models;
+ if (mdl != null)
+ models = stgy.update((GDBModel)mdl, datasetBuilder, featureExtractor, lbExtractor);
+ else
+ models = stgy.learnModels(datasetBuilder, featureExtractor, lbExtractor);
double learningTime = (double)(System.currentTimeMillis() - learningStartTs) / 1000.0;
environment.logger(getClass()).log(MLLogger.VerboseLevel.LOW, "The training time was %.2fs", learningTime);
- return new ModelsComposition(models, resAggregator) {
- @Override public Double apply(Vector features) {
- return internalLabelToExternal(super.apply(features));
- }
- };
- }
-
-
- //TODO: This method will be implemented in IGNITE-9412
- /** {@inheritDoc} */
- @Override public <K, V> ModelsComposition updateModel(ModelsComposition mdl, DatasetBuilder<K, V> datasetBuilder,
- IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
-
- throw new UnsupportedOperationException();
+ WeightedPredictionsAggregator resAggregator = new WeightedPredictionsAggregator(
+ stgy.getCompositionWeights(),
+ stgy.getMeanValue()
+ );
+ return new GDBModel(models, resAggregator, this::internalLabelToExternal);
}
- //TODO: This method will be implemented in IGNITE-9412
/** {@inheritDoc} */
@Override protected boolean checkState(ModelsComposition mdl) {
- throw new UnsupportedOperationException();
+ return mdl instanceof GDBModel;
}
/**
@@ -140,8 +146,9 @@ public abstract class GDBTrainer extends DatasetTrainer<ModelsComposition, Doubl
* @param builder Dataset builder.
* @param featureExtractor Feature extractor.
* @param lExtractor Labels extractor.
+ * @return true if labels learning was successful.
*/
- protected abstract <V, K> void learnLabels(DatasetBuilder<K, V> builder,
+ protected abstract <V, K> boolean learnLabels(DatasetBuilder<K, V> builder,
IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lExtractor);
/**
@@ -196,7 +203,8 @@ public abstract class GDBTrainer extends DatasetTrainer<ModelsComposition, Doubl
}
);
- meanTuple.set1(meanTuple.get1() / meanTuple.get2());
+ if (meanTuple != null)
+ meanTuple.set1(meanTuple.get1() / meanTuple.get2());
return meanTuple;
}
catch (Exception e) {
@@ -205,6 +213,17 @@ public abstract class GDBTrainer extends DatasetTrainer<ModelsComposition, Doubl
}
/**
+ * Sets CheckConvergenceStgyFactory.
+ *
+ * @param factory
+ * @return trainer.
+ */
+ public GDBTrainer withCheckConvergenceStgyFactory(ConvergenceCheckerFactory factory) {
+ this.checkConvergenceStgyFactory = factory;
+ return this;
+ }
+
+ /**
* Returns learning strategy.
*
* @return learning strategy.
@@ -212,4 +231,33 @@ public abstract class GDBTrainer extends DatasetTrainer<ModelsComposition, Doubl
protected GDBLearningStrategy getLearningStrategy() {
return new GDBLearningStrategy();
}
+
+ /** */
+ public static class GDBModel extends ModelsComposition {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 3476661240155508004L;
+
+ /** Internal to external lbl mapping. */
+ private final IgniteFunction<Double, Double> internalToExternalLblMapping;
+
+ /**
+ * Creates an instance of GDBModel.
+ *
+ * @param models Models.
+ * @param predictionsAggregator Predictions aggregator.
+ * @param internalToExternalLblMapping Internal to external lbl mapping.
+ */
+ public GDBModel(List<? extends Model<Vector, Double>> models,
+ WeightedPredictionsAggregator predictionsAggregator,
+ IgniteFunction<Double, Double> internalToExternalLblMapping) {
+
+ super(models, predictionsAggregator);
+ this.internalToExternalLblMapping = internalToExternalLblMapping;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double apply(Vector features) {
+ return internalToExternalLblMapping.apply(super.apply(features));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/LossGradientPerPredictionFunctions.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/LossGradientPerPredictionFunctions.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/LossGradientPerPredictionFunctions.java
deleted file mode 100644
index 488c0e3..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/LossGradientPerPredictionFunctions.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.composition.boosting;
-
-import org.apache.ignite.ml.math.functions.IgniteTriFunction;
-
-/**
- * Contains implementations of per-prediction loss functions for gradient boosting algorithm.
- */
-public class LossGradientPerPredictionFunctions {
- /** Mean squared error loss for regression. */
- public static IgniteTriFunction<Long, Double, Double, Double> MSE =
- (sampleSize, answer, prediction) -> (2.0 / sampleSize) * (prediction - answer);
-
- /** Logarithmic loss for binary classification. */
- public static IgniteTriFunction<Long, Double, Double, Double> LOG_LOSS =
- (sampleSize, answer, prediction) -> (prediction - answer) / (prediction * (1.0 - prediction));
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceChecker.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceChecker.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceChecker.java
new file mode 100644
index 0000000..3f6e8ca
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceChecker.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.convergence;
+
+import java.io.Serializable;
+import org.apache.ignite.ml.composition.ModelsComposition;
+import org.apache.ignite.ml.composition.boosting.loss.Loss;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.primitive.FeatureMatrixWithLabelsOnHeapData;
+import org.apache.ignite.ml.dataset.primitive.FeatureMatrixWithLabelsOnHeapDataBuilder;
+import org.apache.ignite.ml.dataset.primitive.builder.context.EmptyContextBuilder;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+
+/**
+ * Contains logic of error computing and convergence checking for Gradient Boosting algorithms.
+ *
+ * @param <K> Type of a key in upstream data.
+ * @param <V> Type of a value in upstream data.
+ */
+public abstract class ConvergenceChecker<K, V> implements Serializable {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 710762134746674105L;
+
+ /** Sample size. */
+ private long sampleSize;
+
+ /** External label to internal mapping. */
+ private IgniteFunction<Double, Double> externalLbToInternalMapping;
+
+ /** Loss function. */
+ private Loss loss;
+
+ /** Feature extractor. */
+ private IgniteBiFunction<K, V, Vector> featureExtractor;
+
+ /** Label extractor. */
+ private IgniteBiFunction<K, V, Double> lbExtractor;
+
+ /** Precision of convergence check. */
+ private double precision;
+
+ /**
+ * Constructs an instance of ConvergenceChecker.
+ *
+ * @param sampleSize Sample size.
+ * @param externalLbToInternalMapping External label to internal mapping.
+ * @param loss Loss gradient.
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param precision
+ */
+ public ConvergenceChecker(long sampleSize,
+ IgniteFunction<Double, Double> externalLbToInternalMapping, Loss loss,
+ DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor,
+ double precision) {
+
+ assert precision < 1 && precision >= 0;
+
+ this.sampleSize = sampleSize;
+ this.externalLbToInternalMapping = externalLbToInternalMapping;
+ this.loss = loss;
+ this.featureExtractor = featureExtractor;
+ this.lbExtractor = lbExtractor;
+ this.precision = precision;
+ }
+
+ /**
+ * Checks convergency on dataset.
+ *
+ * @param currMdl Current model.
+ * @return true if GDB is converged.
+ */
+ public boolean isConverged(DatasetBuilder<K, V> datasetBuilder, ModelsComposition currMdl) {
+ try (Dataset<EmptyContext, FeatureMatrixWithLabelsOnHeapData> dataset = datasetBuilder.build(
+ new EmptyContextBuilder<>(),
+ new FeatureMatrixWithLabelsOnHeapDataBuilder<>(featureExtractor, lbExtractor)
+ )) {
+ return isConverged(dataset, currMdl);
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Checks convergency on dataset.
+ *
+ * @param dataset Dataset.
+ * @param currMdl Current model.
+ * @return true if GDB is converged.
+ */
+ public boolean isConverged(Dataset<EmptyContext, ? extends FeatureMatrixWithLabelsOnHeapData> dataset, ModelsComposition currMdl) {
+ Double error = computeMeanErrorOnDataset(dataset, currMdl);
+ return error < precision || error.isNaN();
+ }
+
+ /**
+ * Compute error for given model on learning dataset.
+ *
+ * @param dataset Learning dataset.
+ * @param mdl Model.
+ * @return error mean value.
+ */
+ public abstract Double computeMeanErrorOnDataset(
+ Dataset<EmptyContext, ? extends FeatureMatrixWithLabelsOnHeapData> dataset,
+ ModelsComposition mdl);
+
+ /**
+ * Compute error for the specific vector of dataset.
+ *
+ * @param currMdl Current model.
+ * @return error.
+ */
+ public double computeError(Vector features, Double answer, ModelsComposition currMdl) {
+ Double realAnswer = externalLbToInternalMapping.apply(answer);
+ Double mdlAnswer = currMdl.apply(features);
+ return -loss.gradient(sampleSize, realAnswer, mdlAnswer);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceCheckerFactory.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceCheckerFactory.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceCheckerFactory.java
new file mode 100644
index 0000000..7592f50
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/ConvergenceCheckerFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.convergence;
+
+import org.apache.ignite.ml.composition.boosting.loss.Loss;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+
+/**
+ * Factory for ConvergenceChecker.
+ */
+public abstract class ConvergenceCheckerFactory {
+ /** Precision of error checking. If error <= precision then it is equated to 0.0*/
+ protected double precision;
+
+ /**
+ * Creates an instance of ConvergenceCheckerFactory.
+ *
+ * @param precision Precision [0 <= precision < 1].
+ */
+ public ConvergenceCheckerFactory(double precision) {
+ this.precision = precision;
+ }
+
+ /**
+ * Create an instance of ConvergenceChecker.
+ *
+ * @param sampleSize Sample size.
+ * @param externalLbToInternalMapping External label to internal mapping.
+ * @param loss Loss function.
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @return ConvergenceCheckerFactory instance.
+ */
+ public abstract <K,V> ConvergenceChecker<K,V> create(long sampleSize,
+ IgniteFunction<Double, Double> externalLbToInternalMapping, Loss loss,
+ DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceChecker.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceChecker.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceChecker.java
new file mode 100644
index 0000000..7340bfa
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceChecker.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.convergence.mean;
+
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.ml.composition.ModelsComposition;
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceChecker;
+import org.apache.ignite.ml.composition.boosting.loss.Loss;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.primitive.FeatureMatrixWithLabelsOnHeapData;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
+
+/**
+ * Use mean value of errors for estimating error on dataset.
+ *
+ * @param <K> Type of a key in upstream data.
+ * @param <V> Type of a value in upstream data.
+ */
+public class MeanAbsValueConvergenceChecker<K,V> extends ConvergenceChecker<K,V> {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 8534776439755210864L;
+
+ /**
+ * Creates an intance of MeanAbsValueConvergenceChecker.
+ *
+ * @param sampleSize Sample size.
+ * @param externalLbToInternalMapping External label to internal mapping.
+ * @param loss Loss.
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ */
+ public MeanAbsValueConvergenceChecker(long sampleSize, IgniteFunction<Double, Double> externalLbToInternalMapping,
+ Loss loss, DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor,
+ double precision) {
+
+ super(sampleSize, externalLbToInternalMapping, loss, datasetBuilder, featureExtractor, lbExtractor, precision);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double computeMeanErrorOnDataset(Dataset<EmptyContext, ? extends FeatureMatrixWithLabelsOnHeapData> dataset,
+ ModelsComposition mdl) {
+
+ IgniteBiTuple<Double, Long> sumAndCnt = dataset.compute(
+ partition -> computeStatisticOnPartition(mdl, partition),
+ this::reduce
+ );
+
+ if(sumAndCnt == null || sumAndCnt.getValue() == 0)
+ return Double.NaN;
+ return sumAndCnt.getKey() / sumAndCnt.getValue();
+ }
+
+ /**
+ * Compute sum of absolute value of errors and count of rows in partition.
+ *
+ * @param mdl Model.
+ * @param part Partition.
+ * @return Tuple (sum of errors, count of rows)
+ */
+ private IgniteBiTuple<Double, Long> computeStatisticOnPartition(ModelsComposition mdl, FeatureMatrixWithLabelsOnHeapData part) {
+ Double sum = 0.0;
+
+ for(int i = 0; i < part.getFeatures().length; i++) {
+ double error = computeError(VectorUtils.of(part.getFeatures()[i]), part.getLabels()[i], mdl);
+ sum += Math.abs(error);
+ }
+
+ return new IgniteBiTuple<>(sum, (long) part.getLabels().length);
+ }
+
+ /**
+ * Merge left and right statistics from partitions.
+ *
+ * @param left Left.
+ * @param right Right.
+ * @return merged value.
+ */
+ private IgniteBiTuple<Double, Long> reduce(IgniteBiTuple<Double, Long> left, IgniteBiTuple<Double, Long> right) {
+ if (left == null) {
+ if (right != null)
+ return right;
+ else
+ return new IgniteBiTuple<>(0.0, 0L);
+ }
+
+ if (right == null)
+ return left;
+
+ return new IgniteBiTuple<>(
+ left.getKey() + right.getKey(),
+ right.getValue() + left.getValue()
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceCheckerFactory.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceCheckerFactory.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceCheckerFactory.java
new file mode 100644
index 0000000..f02a606
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/mean/MeanAbsValueConvergenceCheckerFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.convergence.mean;
+
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceChecker;
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceCheckerFactory;
+import org.apache.ignite.ml.composition.boosting.loss.Loss;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+
+/**
+ * Factory for {@link MeanAbsValueConvergenceChecker}.
+ */
+public class MeanAbsValueConvergenceCheckerFactory extends ConvergenceCheckerFactory {
+ /**
+ * @param precision Precision.
+ */
+ public MeanAbsValueConvergenceCheckerFactory(double precision) {
+ super(precision);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> ConvergenceChecker<K, V> create(long sampleSize,
+ IgniteFunction<Double, Double> externalLbToInternalMapping, Loss loss, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ return new MeanAbsValueConvergenceChecker<>(sampleSize, externalLbToInternalMapping, loss,
+ datasetBuilder, featureExtractor, lbExtractor, precision);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/mean/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/mean/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/mean/package-info.java
new file mode 100644
index 0000000..1ab6e66
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/mean/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains implementation of convergence checking computer by mean of absolute value of errors in dataset.
+ */
+package org.apache.ignite.ml.composition.boosting.convergence.mean;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceChecker.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceChecker.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceChecker.java
new file mode 100644
index 0000000..7e66a9c
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceChecker.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.convergence.median;
+
+import java.util.Arrays;
+import org.apache.ignite.ml.composition.ModelsComposition;
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceChecker;
+import org.apache.ignite.ml.composition.boosting.loss.Loss;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.primitive.FeatureMatrixWithLabelsOnHeapData;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
+
+/**
+ * Use median of median on partitions value of errors for estimating error on dataset. This algorithm may be less
+ * sensitive to
+ *
+ * @param <K> Type of a key in upstream data.
+ * @param <V> Type of a value in upstream data.
+ */
+public class MedianOfMedianConvergenceChecker<K, V> extends ConvergenceChecker<K, V> {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 4902502002933415287L;
+
+ /**
+ * Creates an instance of MedianOfMedianConvergenceChecker.
+ *
+ * @param sampleSize Sample size.
+ * @param lblMapping External label to internal mapping.
+ * @param loss Loss function.
+ * @param datasetBuilder Dataset builder.
+ * @param fExtr Feature extractor.
+ * @param lbExtr Label extractor.
+ * @param precision Precision.
+ */
+ public MedianOfMedianConvergenceChecker(long sampleSize, IgniteFunction<Double, Double> lblMapping, Loss loss,
+ DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> fExtr,
+ IgniteBiFunction<K, V, Double> lbExtr, double precision) {
+
+ super(sampleSize, lblMapping, loss, datasetBuilder, fExtr, lbExtr, precision);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double computeMeanErrorOnDataset(Dataset<EmptyContext, ? extends FeatureMatrixWithLabelsOnHeapData> dataset,
+ ModelsComposition mdl) {
+
+ double[] medians = dataset.compute(
+ data -> computeMedian(mdl, data),
+ this::reduce
+ );
+
+ if(medians == null)
+ return Double.POSITIVE_INFINITY;
+ return getMedian(medians);
+ }
+
+ /**
+ * Compute median value on data partition.
+ *
+ * @param mdl Model.
+ * @param data Data.
+ * @return median value.
+ */
+ private double[] computeMedian(ModelsComposition mdl, FeatureMatrixWithLabelsOnHeapData data) {
+ double[] errors = new double[data.getLabels().length];
+ for (int i = 0; i < errors.length; i++)
+ errors[i] = Math.abs(computeError(VectorUtils.of(data.getFeatures()[i]), data.getLabels()[i], mdl));
+ return new double[] {getMedian(errors)};
+ }
+
+ /**
+ * Compute median value on array of errors.
+ *
+ * @param errors Error values.
+ * @return median value of errors.
+ */
+ private double getMedian(double[] errors) {
+ if(errors.length == 0)
+ return Double.POSITIVE_INFINITY;
+
+ Arrays.sort(errors);
+ final int middleIdx = (errors.length - 1) / 2;
+ if (errors.length % 2 == 1)
+ return errors[middleIdx];
+ else
+ return (errors[middleIdx + 1] + errors[middleIdx]) / 2;
+ }
+
+ /**
+ * Merge median values among partitions.
+ *
+ * @param left Left partition.
+ * @param right Right partition.
+ * @return merged median values.
+ */
+ private double[] reduce(double[] left, double[] right) {
+ if (left == null)
+ return right;
+ if(right == null)
+ return left;
+
+ double[] res = new double[left.length + right.length];
+ System.arraycopy(left, 0, res, 0, left.length);
+ System.arraycopy(right, 0, res, left.length, right.length);
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceCheckerFactory.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceCheckerFactory.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceCheckerFactory.java
new file mode 100644
index 0000000..a1affe0
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/median/MedianOfMedianConvergenceCheckerFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.convergence.median;
+
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceChecker;
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceCheckerFactory;
+import org.apache.ignite.ml.composition.boosting.loss.Loss;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+
+/**
+ * Factory for {@link MedianOfMedianConvergenceChecker}.
+ */
+public class MedianOfMedianConvergenceCheckerFactory extends ConvergenceCheckerFactory {
+ /**
+ * @param precision Precision.
+ */
+ public MedianOfMedianConvergenceCheckerFactory(double precision) {
+ super(precision);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> ConvergenceChecker<K, V> create(long sampleSize,
+ IgniteFunction<Double, Double> externalLbToInternalMapping, Loss loss, DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ return new MedianOfMedianConvergenceChecker<>(sampleSize, externalLbToInternalMapping, loss,
+ datasetBuilder, featureExtractor, lbExtractor, precision);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/median/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/median/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/median/package-info.java
new file mode 100644
index 0000000..3798ef9
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/median/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains implementation of convergence checking computer by median of medians of errors in dataset.
+ */
+package org.apache.ignite.ml.composition.boosting.convergence.median;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/package-info.java
new file mode 100644
index 0000000..6d42c62
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Package contains implementation of convergency checking algorithms for gradient boosting.
+ * This algorithms may stop training of gradient boosting if it achieve error on dataset less than precision
+ * specified by user.
+ */
+package org.apache.ignite.ml.composition.boosting.convergence;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/simple/ConvergenceCheckerStub.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/simple/ConvergenceCheckerStub.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/simple/ConvergenceCheckerStub.java
new file mode 100644
index 0000000..716d04e
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/simple/ConvergenceCheckerStub.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.convergence.simple;
+
+import org.apache.ignite.ml.composition.ModelsComposition;
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceChecker;
+import org.apache.ignite.ml.composition.boosting.loss.Loss;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.primitive.FeatureMatrixWithLabelsOnHeapData;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+
+/**
+ * This strategy skip estimating error on dataset step.
+ * According to this strategy, training will stop after reaching the maximum number of iterations.
+ *
+ * @param <K> Type of a key in upstream data.
+ * @param <V> Type of a value in upstream data.
+ */
+public class ConvergenceCheckerStub<K,V> extends ConvergenceChecker<K,V> {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 8534776439755210864L;
+
+ /**
+ * Creates an intance of ConvergenceCheckerStub.
+ *
+ * @param sampleSize Sample size.
+ * @param externalLbToInternalMapping External label to internal mapping.
+ * @param loss Loss function.
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ */
+ public ConvergenceCheckerStub(long sampleSize,
+ IgniteFunction<Double, Double> externalLbToInternalMapping, Loss loss,
+ DatasetBuilder<K, V> datasetBuilder,
+ IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ super(sampleSize, externalLbToInternalMapping, loss, datasetBuilder,
+ featureExtractor, lbExtractor, 0.0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isConverged(DatasetBuilder<K, V> datasetBuilder, ModelsComposition currMdl) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isConverged(Dataset<EmptyContext, ? extends FeatureMatrixWithLabelsOnHeapData> dataset,
+ ModelsComposition currMdl) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Double computeMeanErrorOnDataset(Dataset<EmptyContext, ? extends FeatureMatrixWithLabelsOnHeapData> dataset,
+ ModelsComposition mdl) {
+
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/simple/ConvergenceCheckerStubFactory.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/simple/ConvergenceCheckerStubFactory.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/simple/ConvergenceCheckerStubFactory.java
new file mode 100644
index 0000000..a0f0d5c
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/simple/ConvergenceCheckerStubFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.convergence.simple;
+
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceChecker;
+import org.apache.ignite.ml.composition.boosting.convergence.ConvergenceCheckerFactory;
+import org.apache.ignite.ml.composition.boosting.loss.Loss;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.primitives.vector.Vector;
+
+/**
+ * Factory for {@link ConvergenceCheckerStub}.
+ */
+public class ConvergenceCheckerStubFactory extends ConvergenceCheckerFactory {
+ /**
+ * Create an instance of ConvergenceCheckerStubFactory.
+ */
+ public ConvergenceCheckerStubFactory() {
+ super(0.0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> ConvergenceChecker<K, V> create(long sampleSize,
+ IgniteFunction<Double, Double> externalLbToInternalMapping, Loss loss,
+ DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Vector> featureExtractor,
+ IgniteBiFunction<K, V, Double> lbExtractor) {
+
+ return new ConvergenceCheckerStub<>(sampleSize, externalLbToInternalMapping, loss,
+ datasetBuilder, featureExtractor, lbExtractor);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/simple/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/simple/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/simple/package-info.java
new file mode 100644
index 0000000..915903a
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/convergence/simple/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains implementation of Stub for convergence checking.
+ * By this implementation gradient boosting will train new submodels until count of models achieving max value [count
+ * of iterations parameter].
+ */
+package org.apache.ignite.ml.composition.boosting.convergence.simple;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/LogLoss.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/LogLoss.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/LogLoss.java
new file mode 100644
index 0000000..19ef70b
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/LogLoss.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.loss;
+
+/**
+ * Logistic regression loss function.
+ */
+public class LogLoss implements Loss {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 2251384437214194977L;
+
+ /** {@inheritDoc} */
+ @Override public double error(long sampleSize, double answer, double prediction) {
+ return -(answer * Math.log(prediction) + (1 - answer) * Math.log(1 - prediction));
+ }
+
+ /** {@inheritDoc} */
+ @Override public double gradient(long sampleSize, double answer, double prediction) {
+ return (prediction - answer) / (prediction * (1.0 - prediction));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/Loss.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/Loss.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/Loss.java
new file mode 100644
index 0000000..72fff30
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/Loss.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.loss;
+
+import java.io.Serializable;
+
+/**
+ * Loss interface of computing error or gradient of error on specific row in dataset.
+ */
+public interface Loss extends Serializable {
+ /**
+ * Error value for model answer.
+ *
+ * @param sampleSize Sample size.
+ * @param lb Label.
+ * @param mdlAnswer Model answer.
+ * @return error value.
+ */
+ public double error(long sampleSize, double lb, double mdlAnswer);
+
+ /**
+ * Error gradient value for model answer.
+ *
+ * @param sampleSize Sample size.
+ * @param lb Label.
+ * @param mdlAnswer Model answer.
+ * @return error value.
+ */
+ public double gradient(long sampleSize, double lb, double mdlAnswer);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/SquaredError.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/SquaredError.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/SquaredError.java
new file mode 100644
index 0000000..8f2f17e
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/SquaredError.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.composition.boosting.loss;
+
+/**
+ * Represent error function as E(label, modelAnswer) = 1/N * (label - prediction)^2
+ */
+public class SquaredError implements Loss {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 564886150646352157L;
+
+ /** {@inheritDoc} */
+ @Override public double error(long sampleSize, double lb, double prediction) {
+ return Math.pow(lb - prediction, 2) / sampleSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double gradient(long sampleSize, double lb, double prediction) {
+ return (2.0 / sampleSize) * (prediction - lb);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/package-info.java
new file mode 100644
index 0000000..83a5e39
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/boosting/loss/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains loss functions for Gradient Boosting algorithms.
+ */
+package org.apache.ignite.ml.composition.boosting.loss;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/WeightedPredictionsAggregator.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/WeightedPredictionsAggregator.java b/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/WeightedPredictionsAggregator.java
index 8a369ad..5e0f7f1 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/WeightedPredictionsAggregator.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/composition/predictionsaggregator/WeightedPredictionsAggregator.java
@@ -86,4 +86,14 @@ public class WeightedPredictionsAggregator implements PredictionsAggregator {
return builder.append(bias > 0 ? " + " : " - ").append(String.format("%.4f", bias))
.append("]").toString();
}
+
+ /** */
+ public double[] getWeights() {
+ return weights;
+ }
+
+ /** */
+ public double getBias() {
+ return bias;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6225c56e/modules/ml/src/main/java/org/apache/ignite/ml/dataset/primitive/FeatureMatrixWithLabelsOnHeapData.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/primitive/FeatureMatrixWithLabelsOnHeapData.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/primitive/FeatureMatrixWithLabelsOnHeapData.java
new file mode 100644
index 0000000..9dbc1a9
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/primitive/FeatureMatrixWithLabelsOnHeapData.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.dataset.primitive;
+
+/**
+ * A partition {@code data} of the containing matrix of features and vector of labels stored in heap.
+ */
+public class FeatureMatrixWithLabelsOnHeapData implements AutoCloseable {
+ /** Matrix with features. */
+ private final double[][] features;
+
+ /** Vector with labels. */
+ private final double[] labels;
+
+ /**
+ * Constructs an instance of FeatureMatrixWithLabelsOnHeapData.
+ *
+ * @param features Features.
+ * @param labels Labels.
+ */
+ public FeatureMatrixWithLabelsOnHeapData(double[][] features, double[] labels) {
+ assert features.length == labels.length : "Features and labels have to be the same length";
+
+ this.features = features;
+ this.labels = labels;
+ }
+
+ /** */
+ public double[][] getFeatures() {
+ return features;
+ }
+
+ /** */
+ public double[] getLabels() {
+ return labels;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() {
+ // Do nothing, GC will clean up.
+ }
+}
[15/50] [abbrv] ignite git commit: IGNITE-8286: ScanQuery ignore
setLocal with non local partition. - Fixes #3871.
Posted by ag...@apache.org.
IGNITE-8286: ScanQuery ignore setLocal with non local partition. - Fixes #3871.
Signed-off-by: shroman <rs...@yahoo.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e8aa0388
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e8aa0388
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e8aa0388
Branch: refs/heads/ignite-5960
Commit: e8aa038835c235b76bb22580ca479dc679ba5365
Parents: c7aaee7
Author: shroman <rs...@yahoo.com>
Authored: Tue Sep 4 12:32:32 2018 +0900
Committer: shroman <rs...@yahoo.com>
Committed: Tue Sep 4 12:32:32 2018 +0900
----------------------------------------------------------------------
.../processors/cache/IgniteCacheProxyImpl.java | 3 +-
.../cache/query/GridCacheQueryAdapter.java | 30 ++++++++--
.../cache/query/GridCacheQueryManager.java | 8 ++-
...CacheScanPartitionQueryFallbackSelfTest.java | 62 ++++++++++++++++++++
4 files changed, 95 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8aa0388/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index c21ad0b..69ea562 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -379,7 +379,8 @@ public class IgniteCacheProxyImpl<K, V> extends AsyncSupportAdapter<IgniteCache<
IgniteBiPredicate<K, V> p = scanQry.getFilter();
- final CacheQuery<R> qry = ctx.queries().createScanQuery(p, transformer, scanQry.getPartition(), isKeepBinary);
+ final CacheQuery<R> qry = ctx.queries().createScanQuery(
+ p, transformer, scanQry.getPartition(), isKeepBinary, scanQry.isLocal());
if (scanQry.getPageSize() > 0)
qry.pageSize(scanQry.getPageSize());
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8aa0388/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
index 07aea4c..0e3ab43 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
@@ -119,6 +119,9 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
/** */
private volatile boolean incBackups;
+ /** Local query. */
+ private boolean forceLocal;
+
/** */
private volatile boolean dedup;
@@ -143,13 +146,15 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
* @param filter Scan filter.
* @param part Partition.
* @param keepBinary Keep binary flag.
+ * @param forceLocal Flag to force local query.
*/
public GridCacheQueryAdapter(GridCacheContext<?, ?> cctx,
GridCacheQueryType type,
@Nullable IgniteBiPredicate<Object, Object> filter,
@Nullable IgniteClosure<Map.Entry, Object> transform,
@Nullable Integer part,
- boolean keepBinary) {
+ boolean keepBinary,
+ boolean forceLocal) {
assert cctx != null;
assert type != null;
assert part == null || part >= 0;
@@ -160,6 +165,7 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
this.transform = transform;
this.part = part;
this.keepBinary = keepBinary;
+ this.forceLocal = forceLocal;
log = cctx.logger(getClass());
@@ -313,6 +319,13 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
}
/**
+ * @return {@code True} if the query is forced local.
+ */
+ public boolean forceLocal() {
+ return forceLocal;
+ }
+
+ /**
* @return Security subject ID.
*/
public UUID subjectId() {
@@ -516,15 +529,20 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
/** {@inheritDoc} */
@SuppressWarnings({"IfMayBeConditional", "unchecked"})
@Override public GridCloseableIterator executeScanQuery() throws IgniteCheckedException {
- assert type == SCAN : "Wrong processing of qyery: " + type;
+ assert type == SCAN : "Wrong processing of query: " + type;
// Affinity nodes snapshot.
Collection<ClusterNode> nodes = new ArrayList<>(nodes());
cctx.checkSecurity(SecurityPermission.CACHE_READ);
- if (nodes.isEmpty() && part == null)
+ if (nodes.isEmpty()) {
+ if (part != null && forceLocal)
+ throw new IgniteCheckedException("No queryable nodes for partition " + part
+ + " [forced local query=" + this + "]");
+
return new GridEmptyCloseableIterator();
+ }
if (log.isDebugEnabled())
log.debug("Executing query [query=" + this + ", nodes=" + nodes + ']');
@@ -612,9 +630,10 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
* @param cctx Cache context.
* @param prj Projection (optional).
* @return Collection of data nodes in provided projection (if any).
+ * @throws IgniteCheckedException If partition number is invalid.
*/
private static Collection<ClusterNode> nodes(final GridCacheContext<?, ?> cctx,
- @Nullable final ClusterGroup prj, @Nullable final Integer part) {
+ @Nullable final ClusterGroup prj, @Nullable final Integer part) throws IgniteCheckedException {
assert cctx != null;
final AffinityTopologyVersion topVer = cctx.affinity().affinityTopologyVersion();
@@ -624,6 +643,9 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
if (prj == null && part == null)
return affNodes;
+ if (part != null && part >= cctx.affinity().partitions())
+ throw new IgniteCheckedException("Invalid partition number: " + part);
+
final Set<ClusterNode> owners =
part == null ? Collections.<ClusterNode>emptySet() : new HashSet<>(cctx.topology().owners(part, topVer));
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8aa0388/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index c209602..281400e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -2723,7 +2723,7 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
*/
public <R> CacheQuery<R> createScanQuery(@Nullable IgniteBiPredicate<K, V> filter,
@Nullable Integer part, boolean keepBinary) {
- return createScanQuery(filter, null, part, keepBinary);
+ return createScanQuery(filter, null, part, keepBinary, false);
}
/**
@@ -2733,18 +2733,20 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
* @param trans Transformer.
* @param part Partition.
* @param keepBinary Keep binary flag.
+ * @param forceLocal Flag to force local scan.
* @return Created query.
*/
public <T, R> CacheQuery<R> createScanQuery(@Nullable IgniteBiPredicate<K, V> filter,
@Nullable IgniteClosure<T, R> trans,
- @Nullable Integer part, boolean keepBinary) {
+ @Nullable Integer part, boolean keepBinary, boolean forceLocal) {
return new GridCacheQueryAdapter(cctx,
SCAN,
filter,
trans,
part,
- keepBinary);
+ keepBinary,
+ forceLocal);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8aa0388/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java
index 999b1ad..3afcad8 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java
@@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.cache.Cache;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheMode;
@@ -57,6 +58,7 @@ import org.apache.ignite.spi.IgniteSpiException;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
/**
@@ -149,6 +151,66 @@ public class CacheScanPartitionQueryFallbackSelfTest extends GridCommonAbstractT
}
/**
+ * Scan (with explicit {@code setLocal(true)}) should perform on the local node.
+ *
+ * @throws Exception If failed.
+ */
+ public void testScanLocalExplicit() throws Exception {
+ cacheMode = CacheMode.PARTITIONED;
+ backups = 0;
+ commSpiFactory = new TestLocalCommunicationSpiFactory();
+
+ try {
+ Ignite ignite = startGrids(GRID_CNT);
+
+ IgniteCacheProxy<Integer, Integer> cache = fillCache(ignite);
+
+ int part = anyLocalPartition(cache.context());
+
+ QueryCursor<Cache.Entry<Integer, Integer>> qry =
+ cache.query(new ScanQuery<Integer, Integer>().setPartition(part).setLocal(true));
+
+ doTestScanQuery(qry, part);
+
+ GridTestUtils.assertThrows(log, (Callable<Void>)() -> {
+ int remPart = remotePartition(cache.context()).getKey();
+
+ cache.query(new ScanQuery<Integer, Integer>().setPartition(remPart).setLocal(true));
+
+ return null;
+ }, IgniteCheckedException.class, null);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * Scan (with explicit {@code setLocal(true)}, no partition specified) should perform on the local node.
+ *
+ * @throws Exception If failed.
+ */
+ public void testScanLocalExplicitNoPart() throws Exception {
+ cacheMode = CacheMode.PARTITIONED;
+ backups = 0;
+ commSpiFactory = new TestLocalCommunicationSpiFactory();
+
+ try {
+ Ignite ignite = startGrids(GRID_CNT);
+
+ IgniteCacheProxy<Integer, Integer> cache = fillCache(ignite);
+
+ QueryCursor<Cache.Entry<Integer, Integer>> qry =
+ cache.query(new ScanQuery<Integer, Integer>().setLocal(true));
+
+ assertFalse(qry.getAll().isEmpty());
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
* Scan should perform on the remote node.
*
* @throws Exception If failed.
[31/50] [abbrv] ignite git commit: IGNITE-9438 Fix file descriptors
leak in StandaloneWalRecordsIterator. - Fixes #4658.
Posted by ag...@apache.org.
IGNITE-9438 Fix file descriptors leak in StandaloneWalRecordsIterator. - Fixes #4658.
Signed-off-by: Dmitriy Govorukhin <dm...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ef4a02dc
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ef4a02dc
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ef4a02dc
Branch: refs/heads/ignite-5960
Commit: ef4a02dc5d3f66258afa9a3ded8d7671adf6ee73
Parents: 9862dbe
Author: Sergey Antonov <an...@gmail.com>
Authored: Wed Sep 5 11:33:07 2018 +0300
Committer: Dmitriy Govorukhin <dm...@gmail.com>
Committed: Wed Sep 5 11:33:07 2018 +0300
----------------------------------------------------------------------
.../wal/AbstractWalRecordsIterator.java | 139 ++++++++----
.../reader/StandaloneWalRecordsIterator.java | 33 +--
.../ignite/internal/util/IgniteUtils.java | 16 ++
.../StandaloneWalRecordsIteratorTest.java | 216 +++++++++++++++++++
.../ignite/testsuites/IgnitePdsTestSuite2.java | 8 +-
5 files changed, 349 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4a02dc/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index 9fbb535..0b704ca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -36,6 +36,7 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.Re
import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.SegmentHeader;
import org.apache.ignite.internal.util.GridCloseableIteratorAdapter;
import org.apache.ignite.internal.util.typedef.P2;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -43,8 +44,8 @@ import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readSegmentHeader;
/**
- * Iterator over WAL segments. This abstract class provides most functionality for reading records in log.
- * Subclasses are to override segment switching functionality
+ * Iterator over WAL segments. This abstract class provides most functionality for reading records in log. Subclasses
+ * are to override segment switching functionality
*/
public abstract class AbstractWalRecordsIterator
extends GridCloseableIteratorAdapter<IgniteBiTuple<WALPointer, WALRecord>> implements WALIterator {
@@ -52,14 +53,14 @@ public abstract class AbstractWalRecordsIterator
private static final long serialVersionUID = 0L;
/**
- * Current record preloaded, to be returned on next()<br>
- * Normally this should be not null because advance() method should already prepare some value<br>
+ * Current record preloaded, to be returned on next()<br> Normally this should be not null because advance() method
+ * should already prepare some value<br>
*/
protected IgniteBiTuple<WALPointer, WALRecord> curRec;
/**
- * Current WAL segment absolute index. <br>
- * Determined as lowest number of file at start, is changed during advance segment
+ * Current WAL segment absolute index. <br> Determined as lowest number of file at start, is changed during advance
+ * segment
*/
protected long curWalSegmIdx = -1;
@@ -179,7 +180,6 @@ public abstract class AbstractWalRecordsIterator
}
/**
- *
* @param tailReachedException Tail reached exception.
* @param currWalSegment Current WAL segment read handler.
* @return If need to throw exception after validation.
@@ -210,9 +210,8 @@ public abstract class AbstractWalRecordsIterator
}
/**
- * Switches records iterator to the next WAL segment
- * as result of this method, new reference to segment should be returned.
- * Null for current handle means stop of iteration.
+ * Switches records iterator to the next WAL segment as result of this method, new reference to segment should be
+ * returned. Null for current handle means stop of iteration.
*
* @param curWalSegment current open WAL segment or null if there is no open segment yet
* @return new WAL segment to read or null for stop iteration
@@ -263,8 +262,8 @@ public abstract class AbstractWalRecordsIterator
}
/**
- * Performs final conversions with record loaded from WAL.
- * To be overridden by subclasses if any processing required.
+ * Performs final conversions with record loaded from WAL. To be overridden by subclasses if any processing
+ * required.
*
* @param rec record to post process.
* @return post processed record.
@@ -278,11 +277,11 @@ public abstract class AbstractWalRecordsIterator
*
* @param e problem from records reading
* @param ptr file pointer was accessed
- *
- * @return {@code null} if the error was handled and we can go ahead,
- * {@code IgniteCheckedException} if the error was not handled, and we should stop the iteration.
+ * @return {@code null} if the error was handled and we can go ahead, {@code IgniteCheckedException} if the error
+ * was not handled, and we should stop the iteration.
*/
- protected IgniteCheckedException handleRecordException(@NotNull final Exception e, @Nullable final FileWALPointer ptr) {
+ protected IgniteCheckedException handleRecordException(@NotNull final Exception e,
+ @Nullable final FileWALPointer ptr) {
if (log.isInfoEnabled())
log.info("Stopping WAL iteration due to an exception: " + e.getMessage() + ", ptr=" + ptr);
@@ -290,45 +289,92 @@ public abstract class AbstractWalRecordsIterator
}
/**
+ * Assumes fileIO will be closed in this method in case of error occurred.
+ *
* @param desc File descriptor.
- * @param start Optional start pointer. Null means read from the beginning
- * @return Initialized file handle.
- * @throws FileNotFoundException If segment file is missing.
+ * @param start Optional start pointer. Null means read from the beginning.
+ * @param fileIO fileIO associated with file descriptor
+ * @param segmentHeader read segment header from fileIO
+ * @return Initialized file read header.
* @throws IgniteCheckedException If initialized failed due to another unexpected error.
*/
protected AbstractReadFileHandle initReadHandle(
@NotNull final AbstractFileDescriptor desc,
- @Nullable final FileWALPointer start
- ) throws IgniteCheckedException, FileNotFoundException {
+ @Nullable final FileWALPointer start,
+ @NotNull final FileIO fileIO,
+ @NotNull final SegmentHeader segmentHeader
+ ) throws IgniteCheckedException {
try {
- FileIO fileIO = desc.isCompressed() ? new UnzipFileIO(desc.file()) : ioFactory.create(desc.file());
+ final boolean isCompacted = segmentHeader.isCompacted();
+
+ if (isCompacted)
+ serializerFactory.skipPositionCheck(true);
+
+ FileInput in = new FileInput(fileIO, buf);
+
+ if (start != null && desc.idx() == start.index()) {
+ if (isCompacted) {
+ if (start.fileOffset() != 0)
+ serializerFactory.recordDeserializeFilter(new StartSeekingFilter(start));
+ }
+ else {
+ // Make sure we skip header with serializer version.
+ long startOff = Math.max(start.fileOffset(), fileIO.position());
+ in.seek(startOff);
+ }
+ }
+
+ int serVer = segmentHeader.getSerializerVersion();
+
+ return createReadFileHandle(fileIO, desc.idx(), serializerFactory.createSerializer(serVer), in);
+ }
+ catch (SegmentEofException | EOFException ignore) {
try {
- SegmentHeader segmentHeader = readSegmentHeader(fileIO, curWalSegmIdx);
+ fileIO.close();
+ }
+ catch (IOException ce) {
+ throw new IgniteCheckedException(ce);
+ }
- boolean isCompacted = segmentHeader.isCompacted();
+ return null;
+ }
+ catch (IgniteCheckedException e) {
+ U.closeWithSuppressingException(fileIO, e);
- if (isCompacted)
- serializerFactory.skipPositionCheck(true);
+ throw e;
+ }
+ catch (IOException e) {
+ U.closeWithSuppressingException(fileIO, e);
- FileInput in = new FileInput(fileIO, buf);
+ throw new IgniteCheckedException(
+ "Failed to initialize WAL segment after reading segment header: " + desc.file().getAbsolutePath(), e);
+ }
+ }
- if (start != null && desc.idx() == start.index()) {
- if (isCompacted) {
- if (start.fileOffset() != 0)
- serializerFactory.recordDeserializeFilter(new StartSeekingFilter(start));
- }
- else {
- // Make sure we skip header with serializer version.
- long startOff = Math.max(start.fileOffset(), fileIO.position());
+ /**
+ * Assumes file descriptor will be opened in this method. The caller of this method must be responsible for closing
+ * opened file descriptor File descriptor will be closed ONLY in case of error occurred.
+ *
+ * @param desc File descriptor.
+ * @param start Optional start pointer. Null means read from the beginning
+ * @return Initialized file read header.
+ * @throws FileNotFoundException If segment file is missing.
+ * @throws IgniteCheckedException If initialized failed due to another unexpected error.
+ */
+ protected AbstractReadFileHandle initReadHandle(
+ @NotNull final AbstractFileDescriptor desc,
+ @Nullable final FileWALPointer start
+ ) throws IgniteCheckedException, FileNotFoundException {
+ FileIO fileIO = null;
- in.seek(startOff);
- }
- }
+ try {
+ fileIO = desc.isCompressed() ? new UnzipFileIO(desc.file()) : ioFactory.create(desc.file());
- int serVer = segmentHeader.getSerializerVersion();
+ SegmentHeader segmentHeader;
- return createReadFileHandle(fileIO, desc.idx(), serializerFactory.createSerializer(serVer), in);
+ try {
+ segmentHeader = readSegmentHeader(fileIO, curWalSegmIdx);
}
catch (SegmentEofException | EOFException ignore) {
try {
@@ -341,20 +387,21 @@ public abstract class AbstractWalRecordsIterator
return null;
}
catch (IOException | IgniteCheckedException e) {
- try {
- fileIO.close();
- }
- catch (IOException ce) {
- e.addSuppressed(ce);
- }
+ U.closeWithSuppressingException(fileIO, e);
throw e;
}
+
+ return initReadHandle(desc, start, fileIO, segmentHeader);
}
catch (FileNotFoundException e) {
+ U.closeQuiet(fileIO);
+
throw e;
}
catch (IOException e) {
+ U.closeQuiet(fileIO);
+
throw new IgniteCheckedException(
"Failed to initialize WAL segment: " + desc.file().getAbsolutePath(), e);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4a02dc/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index 6729790..a222877 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -48,8 +48,10 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAhea
import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException;
import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl;
+import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.SegmentHeader;
import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiPredicate;
import org.apache.ignite.lang.IgniteBiTuple;
import org.jetbrains.annotations.NotNull;
@@ -58,8 +60,8 @@ import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readSegmentHeader;
/**
- * WAL reader iterator, for creation in standalone WAL reader tool
- * Operates over one directory, does not provide start and end boundaries
+ * WAL reader iterator, for creation in standalone WAL reader tool Operates over one directory, does not provide start
+ * and end boundaries
*/
class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
/** Record buffer size */
@@ -88,12 +90,13 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
/**
* Creates iterator in file-by-file iteration mode. Directory
+ *
* @param log Logger.
* @param sharedCtx Shared context. Cache processor is to be configured if Cache Object Key & Data Entry is
* required.
* @param ioFactory File I/O factory.
- * @param keepBinary Keep binary. This flag disables converting of non primitive types
- * (BinaryObjects will be used instead)
+ * @param keepBinary Keep binary. This flag disables converting of non primitive types (BinaryObjects will be used
+ * instead)
* @param walFiles Wal files.
*/
StandaloneWalRecordsIterator(
@@ -128,8 +131,8 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
}
/**
- * For directory mode sets oldest file as initial segment,
- * for file by file mode, converts all files to descriptors and gets oldest as initial.
+ * For directory mode sets oldest file as initial segment, for file by file mode, converts all files to descriptors
+ * and gets oldest as initial.
*
* @param walFiles files for file-by-file iteration mode
*/
@@ -232,7 +235,6 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
}
/**
- *
* @param ptr WAL pointer.
* @return {@code True} If pointer between low and high bounds. {@code False} if not.
*/
@@ -243,7 +245,6 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
}
/**
- *
* @param idx WAL segment index.
* @return {@code True} If pointer between low and high bounds. {@code False} if not.
*/
@@ -258,18 +259,21 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
) throws IgniteCheckedException, FileNotFoundException {
AbstractFileDescriptor fd = desc;
-
+ FileIO fileIO = null;
+ SegmentHeader segmentHeader;
while (true) {
try {
- FileIO fileIO = fd.isCompressed() ? new UnzipFileIO(fd.file()) : ioFactory.create(fd.file());
+ fileIO = fd.isCompressed() ? new UnzipFileIO(fd.file()) : ioFactory.create(fd.file());
- readSegmentHeader(fileIO, curWalSegmIdx);
+ segmentHeader = readSegmentHeader(fileIO, curWalSegmIdx);
break;
}
catch (IOException | IgniteCheckedException e) {
log.error("Failed to init segment curWalSegmIdx=" + curWalSegmIdx + ", curIdx=" + curIdx, e);
+ U.closeQuiet(fileIO);
+
curIdx++;
if (curIdx >= walFileDescriptors.size())
@@ -279,13 +283,13 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
}
}
- return super.initReadHandle(fd, start);
+ return initReadHandle(fd, start, fileIO, segmentHeader);
}
/** {@inheritDoc} */
@NotNull @Override protected WALRecord postProcessRecord(@NotNull final WALRecord rec) {
- GridKernalContext kernalCtx = sharedCtx.kernalContext();
- IgniteCacheObjectProcessor processor = kernalCtx.cacheObjects();
+ GridKernalContext kernalCtx = sharedCtx.kernalContext();
+ IgniteCacheObjectProcessor processor = kernalCtx.cacheObjects();
if (processor != null && rec.type() == RecordType.DATA_RECORD) {
try {
@@ -335,6 +339,7 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
/**
* Converts entry or lazy data entry into unwrapped entry
+ *
* @param processor cache object processor for de-serializing objects.
* @param fakeCacheObjCtx cache object context for de-serializing binary and unwrapping objects.
* @param dataEntry entry to process
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4a02dc/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index a68fb6d..d2d2b89 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -4020,6 +4020,22 @@ public abstract class IgniteUtils {
}
/**
+ * Closes given resource suppressing possible checked exception.
+ *
+ * @param rsrc Resource to close. If it's {@code null} - it's no-op.
+ * @param e Suppressor exception
+ */
+ public static void closeWithSuppressingException(@Nullable AutoCloseable rsrc, @NotNull Exception e) {
+ if (rsrc != null)
+ try {
+ rsrc.close();
+ }
+ catch (Exception suppressed) {
+ e.addSuppressed(suppressed);
+ }
+ }
+
+ /**
* Quietly closes given resource ignoring possible checked exception.
*
* @param rsrc Resource to close. If it's {@code null} - it's no-op.
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4a02dc/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
new file mode 100644
index 0000000..b6a04d0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.reader;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.OpenOption;
+import java.nio.file.StandardOpenOption;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.record.SnapshotRecord;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder;
+
+/**
+ * The test check, that StandaloneWalRecordsIterator correctly close file descriptors associated with WAL files.
+ */
+public class StandaloneWalRecordsIteratorTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(name);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration().
+ setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ )
+ ).setDiscoverySpi(
+ new TcpDiscoverySpi()
+ .setIpFinder(IP_FINDER)
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * Check correct closing file descriptors.
+ *
+ * @throws Exception if test failed.
+ */
+ public void testCorrectClosingFileDescriptors() throws Exception {
+ IgniteEx ig = (IgniteEx)startGrid();
+
+ String archiveWalDir = getArchiveWalDirPath(ig);
+
+ ig.cluster().active(true);
+
+ IgniteCacheDatabaseSharedManager sharedMgr = ig.context().cache().context().database();
+
+ IgniteWriteAheadLogManager walMgr = ig.context().cache().context().wal();
+
+ // Generate WAL segments for filling WAL archive folder.
+ for (int i = 0; i < 2 * ig.configuration().getDataStorageConfiguration().getWalSegments(); i++) {
+ sharedMgr.checkpointReadLock();
+
+ try {
+ walMgr.log(new SnapshotRecord(i, false));
+ }
+ finally {
+ sharedMgr.checkpointReadUnlock();
+ }
+ }
+
+ stopGrid();
+
+ // Iterate by all archived WAL segments.
+ createWalIterator(archiveWalDir).forEach(x -> {
+ });
+
+ assertTrue("At least one WAL file must be opened!", CountedFileIO.getCountOpenedWalFiles() > 0);
+
+ assertEquals("All WAL files must be closed!", CountedFileIO.getCountOpenedWalFiles(), CountedFileIO.getCountClosedWalFiles());
+ }
+
+ /**
+ * Creates WALIterator associated with files inside walDir.
+ *
+ * @param walDir - path to WAL directory.
+ * @return WALIterator associated with files inside walDir.
+ * @throws IgniteCheckedException if error occur.
+ */
+ private WALIterator createWalIterator(String walDir) throws IgniteCheckedException {
+ IteratorParametersBuilder params = new IteratorParametersBuilder();
+
+ params.ioFactory(new CountedFileIOFactory());
+
+ return new IgniteWalIteratorFactory(log).iterator(params.filesOrDirs(walDir));
+ }
+
+ /**
+ * Evaluate path to directory with WAL archive.
+ *
+ * @param ignite instance of Ignite.
+ * @return path to directory with WAL archive.
+ * @throws IgniteCheckedException if error occur.
+ */
+ private String getArchiveWalDirPath(Ignite ignite) throws IgniteCheckedException {
+ return U.resolveWorkDirectory(
+ U.defaultWorkDirectory(),
+ ignite.configuration().getDataStorageConfiguration().getWalArchivePath(),
+ false
+ ).getAbsolutePath();
+ }
+
+ /**
+ *
+ */
+ private static class CountedFileIOFactory extends RandomAccessFileIOFactory {
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file) throws IOException {
+ return create(file, StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE);
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+ return new CountedFileIO(file, modes);
+ }
+ }
+
+ /**
+ *
+ */
+ private static class CountedFileIO extends RandomAccessFileIO {
+ /** Wal open counter. */
+ private static final AtomicInteger WAL_OPEN_COUNTER = new AtomicInteger();
+ /** Wal close counter. */
+ private static final AtomicInteger WAL_CLOSE_COUNTER = new AtomicInteger();
+
+ /** File name. */
+ private final String fileName;
+
+ /** */
+ public CountedFileIO(File file, OpenOption... modes) throws IOException {
+ super(file, modes);
+
+ fileName = file.getName();
+
+ if (FileWriteAheadLogManager.WAL_NAME_PATTERN.matcher(fileName).matches())
+ WAL_OPEN_COUNTER.incrementAndGet();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IOException {
+ super.close();
+
+ if (FileWriteAheadLogManager.WAL_NAME_PATTERN.matcher(fileName).matches())
+ WAL_CLOSE_COUNTER.incrementAndGet();
+ }
+
+ /**
+ *
+ * @return number of opened files.
+ */
+ public static int getCountOpenedWalFiles() { return WAL_OPEN_COUNTER.get(); }
+
+ /**
+ *
+ * @return number of closed files.
+ */
+ public static int getCountClosedWalFiles() { return WAL_CLOSE_COUNTER.get(); }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4a02dc/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 5eba5a4..956d256 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -34,7 +34,6 @@ import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOf
import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOnlineNodeOutOfBaselineFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsReserveWalSegmentsTest;
-import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest;
import org.apache.ignite.internal.processors.cache.persistence.db.SlowHistoricalRebalanceSmallHistoryTest;
import org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest;
@@ -53,13 +52,13 @@ import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalF
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalHistoryReservationsTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorExceptionDuringReadTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorSwitchSegmentTest;
-import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoverySeveralRestartsTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalSerializerVersionTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalCompactionTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalDeletionArchiveFsyncTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalDeletionArchiveLogOnlyTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteDataIntegrityTests;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.reader.IgniteWalReaderTest;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneWalRecordsIteratorTest;
/**
*
@@ -89,7 +88,8 @@ public class IgnitePdsTestSuite2 extends TestSuite {
}
/**
- * Fills {@code suite} with PDS test subset, which operates with real page store, but requires long time to execute.
+ * Fills {@code suite} with PDS test subset, which operates with real page store, but requires long time to
+ * execute.
*
* @param suite suite to add tests into.
*/
@@ -171,6 +171,8 @@ public class IgnitePdsTestSuite2 extends TestSuite {
suite.addTestSuite(IgniteNodeStoppedDuringDisableWALTest.class);
+ suite.addTestSuite(StandaloneWalRecordsIteratorTest.class);
+
//suite.addTestSuite(IgniteWalRecoverySeveralRestartsTest.class);
suite.addTestSuite(IgniteRebalanceScheduleResendPartitionsTest.class);
[45/50] [abbrv] ignite git commit: IGNITE-8927: SQL: correct handling
of LOST partitions on mapper side. Note that for local queries this doesn't
work still because partition state is not checked (see IGNITE-7039). This
closes #4679.
Posted by ag...@apache.org.
IGNITE-8927: SQL: correct handling of LOST partitions on mapper side. Note that for local queries this doesn't work still because partition state is not checked (see IGNITE-7039). This closes #4679.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0a23658e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0a23658e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0a23658e
Branch: refs/heads/ignite-5960
Commit: 0a23658e2d5f12420ab021aa1d3ee1d08506d2f6
Parents: 469fb49
Author: devozerov <vo...@gridgain.com>
Authored: Fri Sep 7 12:19:55 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Sep 7 12:19:55 2018 +0300
----------------------------------------------------------------------
.../twostep/messages/GridQueryFailResponse.java | 2 +-
.../IgniteCachePartitionLossPolicySelfTest.java | 29 +++-
.../query/h2/twostep/GridMapQueryExecutor.java | 123 +++++++++++-----
.../h2/twostep/GridReduceQueryExecutor.java | 2 +-
...ndexingCachePartitionLossPolicySelfTest.java | 139 +++++++++++++++++++
...niteCacheDistributedQueryCancelSelfTest.java | 2 +-
.../h2/twostep/RetryCauseMessageSelfTest.java | 3 +-
.../IgniteCacheQuerySelfTestSuite.java | 4 +
8 files changed, 261 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0a23658e/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryFailResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryFailResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryFailResponse.java
index 1b759bb..ef26d2a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryFailResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryFailResponse.java
@@ -59,7 +59,7 @@ public class GridQueryFailResponse implements Message {
*/
public GridQueryFailResponse(long qryReqId, Throwable err) {
this.qryReqId = qryReqId;
- this.errMsg = err.getClass() + ":" + err.getMessage();
+ this.errMsg = err.getMessage();
this.failCode = err instanceof QueryCancelledException ? CANCELLED_BY_ORIGINATOR : GENERAL_ERROR;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0a23658e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
index 0be8526..1aacc9c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
@@ -72,7 +72,7 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe
private PartitionLossPolicy partLossPlc;
/** */
- private static final String CACHE_NAME = "partitioned";
+ protected static final String CACHE_NAME = "partitioned";
/** */
private int backups = 0;
@@ -101,6 +101,15 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe
cfg.setClientMode(client);
+ cfg.setCacheConfiguration(cacheConfiguration());
+
+ return cfg;
+ }
+
+ /**
+ * @return Cache configuration.
+ */
+ protected CacheConfiguration<Integer, Integer> cacheConfiguration() {
CacheConfiguration<Integer, Integer> cacheCfg = new CacheConfiguration<>(CACHE_NAME);
cacheCfg.setCacheMode(PARTITIONED);
@@ -109,9 +118,7 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe
cacheCfg.setPartitionLossPolicy(partLossPlc);
cacheCfg.setAffinity(new RendezvousAffinityFunction(false, 32));
- cfg.setCacheConfiguration(cacheCfg);
-
- return cfg;
+ return cacheCfg;
}
/** {@inheritDoc} */
@@ -294,6 +301,9 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe
// Check that writing in recover mode does not clear partition state.
verifyCacheOps(canWrite, safe, part, ig);
+
+ // Validate queries.
+ validateQuery(safe, part, ig);
}
// Check that partition state does not change after we start a new node.
@@ -410,6 +420,17 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe
return parts;
}
+ /**
+ * Validate query execution on a node.
+ *
+ * @param safe Safe flag.
+ * @param part Partition.
+ * @param node Node.
+ */
+ protected void validateQuery(boolean safe, int part, Ignite node) {
+ // No-op.
+ }
+
/** */
class TopologyChanger {
/** Flag to delay partition exchange */
http://git-wip-us.apache.org/repos/asf/ignite/blob/0a23658e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index b4d6f00..9166604 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -41,6 +41,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.PartitionLossPolicy;
import org.apache.ignite.cache.query.QueryCancelledException;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cluster.ClusterNode;
@@ -54,6 +55,7 @@ import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.managers.communication.GridMessageListener;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.distributed.dht.CompoundLockFuture;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
@@ -102,9 +104,12 @@ import org.h2.value.Value;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_FORCE_LAZY_RESULT_SET;
+import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_SAFE;
+import static org.apache.ignite.cache.PartitionLossPolicy.READ_WRITE_SAFE;
import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED;
import static org.apache.ignite.internal.managers.communication.GridIoPolicy.QUERY_POOL;
import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.LOST;
import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.OFF;
import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.distributedJoinMode;
@@ -335,11 +340,9 @@ public class GridMapQueryExecutor {
// Cache was not found, probably was not deployed yet.
if (cctx == null) {
- final String res = String.format("Failed to reserve partitions for query (cache is not found on " +
+ return String.format("Failed to reserve partitions for query (cache is not found on " +
"local node) [localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s]",
ctx.localNodeId(), nodeId, reqId, topVer, cacheIds.get(i));
-
- return res;
}
if (cctx.isLocal() || !cctx.rebalanceEnabled())
@@ -396,16 +399,39 @@ public class GridMapQueryExecutor {
if (explicitParts == null)
partIds = cctx.affinity().primaryPartitions(ctx.localNodeId(), topVer);
+ int reservedCnt = 0;
+
for (int partId : partIds) {
GridDhtLocalPartition part = partition(cctx, partId);
GridDhtPartitionState partState = part != null ? part.state() : null;
- if (partState != OWNING || !part.reserve())
+ if (partState != OWNING) {
+ if (partState == LOST)
+ ignoreLostPartitionIfPossible(cctx, part);
+ else {
+ return String.format("Failed to reserve partitions for query " +
+ "(partition of PARTITIONED cache is not found or not in OWNING state) [" +
+ "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, " +
+ "cacheName=%s, part=%s, partFound=%s, partState=%s]",
+ ctx.localNodeId(),
+ nodeId,
+ reqId,
+ topVer,
+ cacheIds.get(i),
+ cctx.name(),
+ partId,
+ (part != null),
+ partState
+ );
+ }
+ }
+
+ if (!part.reserve()) {
return String.format("Failed to reserve partitions for query " +
- "(partition of PARTITIONED cache cannot be reserved) [" +
- "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, cacheName=%s, " +
- "part=%s, partFound=%s, partState=%s]",
+ "(partition of PARTITIONED cache cannot be reserved) [" +
+ "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, " +
+ "cacheName=%s, part=%s, partFound=%s, partState=%s]",
ctx.localNodeId(),
nodeId,
reqId,
@@ -413,36 +439,44 @@ public class GridMapQueryExecutor {
cacheIds.get(i),
cctx.name(),
partId,
- (part != null),
+ true,
partState
);
+ }
reserved.add(part);
+ reservedCnt++;
+
// Double check that we are still in owning state and partition contents are not cleared.
partState = part.state();
- if (part.state() != OWNING)
- return String.format("Failed to reserve partitions for query " +
- "(partition of PARTITIONED cache is not in OWNING state after reservation) [" +
- "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, cacheName=%s, " +
- "part=%s, partState=%s]",
- ctx.localNodeId(),
- nodeId,
- reqId,
- topVer,
- cacheIds.get(i),
- cctx.name(),
- partId,
- partState
- );
+ if (partState != OWNING) {
+ if (partState == LOST)
+ ignoreLostPartitionIfPossible(cctx, part);
+ else {
+ return String.format("Failed to reserve partitions for query " +
+ "(partition of PARTITIONED cache is not in OWNING state after reservation) [" +
+ "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, " +
+ "cacheName=%s, part=%s, partState=%s]",
+ ctx.localNodeId(),
+ nodeId,
+ reqId,
+ topVer,
+ cacheIds.get(i),
+ cctx.name(),
+ partId,
+ partState
+ );
+ }
+ }
}
- if (explicitParts == null) {
+ if (explicitParts == null && reservedCnt > 0) {
// We reserved all the primary partitions for cache, attempt to add group reservation.
GridDhtPartitionsReservation grp = new GridDhtPartitionsReservation(topVer, cctx, "SQL");
- if (grp.register(reserved.subList(reserved.size() - partIds.size(), reserved.size()))) {
+ if (grp.register(reserved.subList(reserved.size() - reservedCnt, reserved.size()))) {
if (reservations.putIfAbsent(grpKey, grp) != null)
throw new IllegalStateException("Reservation already exists.");
@@ -461,6 +495,25 @@ public class GridMapQueryExecutor {
}
/**
+ * Decide whether to ignore or proceed with lost partition.
+ *
+ * @param cctx Cache context.
+ * @param part Partition.
+ * @throws IgniteCheckedException If failed.
+ */
+ private static void ignoreLostPartitionIfPossible(GridCacheContext cctx, GridDhtLocalPartition part)
+ throws IgniteCheckedException {
+ PartitionLossPolicy plc = cctx.config().getPartitionLossPolicy();
+
+ if (plc != null) {
+ if (plc == READ_ONLY_SAFE || plc == READ_WRITE_SAFE) {
+ throw new CacheInvalidStateException("Failed to execute query because cache partition has been " +
+ "lost [cacheName=" + cctx.name() + ", part=" + part + ']');
+ }
+ }
+ }
+
+ /**
* @param ints Integers.
* @return Collection wrapper.
*/
@@ -1045,22 +1098,22 @@ public class GridMapQueryExecutor {
List<GridReservable> reserved = new ArrayList<>();
- String err = reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId);
+ MapNodeResults nodeResults = resultsForNode(node.id());
- if (!F.isEmpty(err)) {
- U.error(log, "Failed to reserve partitions for DML request. [localNodeId=" + ctx.localNodeId() +
- ", nodeId=" + node.id() + ", reqId=" + req.requestId() + ", cacheIds=" + cacheIds +
- ", topVer=" + topVer + ", parts=" + Arrays.toString(parts) + ']');
+ try {
+ String err = reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId);
- sendUpdateResponse(node, reqId, null,
- "Failed to reserve partitions for DML request. " + err);
+ if (!F.isEmpty(err)) {
+ U.error(log, "Failed to reserve partitions for DML request. [localNodeId=" + ctx.localNodeId() +
+ ", nodeId=" + node.id() + ", reqId=" + req.requestId() + ", cacheIds=" + cacheIds +
+ ", topVer=" + topVer + ", parts=" + Arrays.toString(parts) + ']');
- return;
- }
+ sendUpdateResponse(node, reqId, null,
+ "Failed to reserve partitions for DML request. " + err);
- MapNodeResults nodeResults = resultsForNode(node.id());
+ return;
+ }
- try {
IndexingQueryFilter filter = h2.backupFilter(topVer, parts);
GridQueryCancel cancel = nodeResults.putUpdate(reqId);
http://git-wip-us.apache.org/repos/asf/ignite/blob/0a23658e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 586fb51..96c88ff 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -884,7 +884,7 @@ public class GridReduceQueryExecutor {
if (wasCancelled(err))
throw new QueryCancelledException(); // Throw correct exception.
- throw new CacheException("Failed to run map query remotely: " + err.getMessage(), err);
+ throw err;
}
else {
retry = true;
http://git-wip-us.apache.org/repos/asf/ignite/blob/0a23658e/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java
new file mode 100644
index 0000000..f208599
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePartitionLossPolicySelfTest;
+
+import java.util.Collection;
+
+/**
+ * Partition loss policy test with enabled indexing.
+ */
+public class IndexingCachePartitionLossPolicySelfTest extends IgniteCachePartitionLossPolicySelfTest {
+ /** {@inheritDoc} */
+ @Override protected CacheConfiguration<Integer, Integer> cacheConfiguration() {
+ CacheConfiguration<Integer, Integer> ccfg = super.cacheConfiguration();
+
+ ccfg.setIndexedTypes(Integer.class, Integer.class);
+
+ return ccfg;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override protected void validateQuery(boolean safe, int part, Ignite node) {
+ // Get node lost and remaining partitions.
+ IgniteCache cache = node.cache(CACHE_NAME);
+
+ Collection<Integer> lostParts = cache.lostPartitions();
+
+ Integer remainingPart = null;
+
+ for (int i = 0; i < node.affinity(CACHE_NAME).partitions(); i++) {
+ if (lostParts.contains(i))
+ continue;
+
+ remainingPart = i;
+
+ break;
+ }
+
+ // Determine whether local query should be executed on that node.
+ boolean execLocQry = false;
+
+ for (int nodePrimaryPart : node.affinity(CACHE_NAME).primaryPartitions(node.cluster().localNode())) {
+ if (part == nodePrimaryPart) {
+ execLocQry = true;
+
+ break;
+ }
+ }
+
+ // 1. Check query against all partitions.
+ validateQuery0(safe, node, false);
+
+ // TODO: https://issues.apache.org/jira/browse/IGNITE-7039
+// if (execLocQry)
+// validateQuery0(safe, node, true);
+
+ // 2. Check query against LOST partition.
+ validateQuery0(safe, node, false, part);
+
+ // TODO: https://issues.apache.org/jira/browse/IGNITE-7039
+// if (execLocQry)
+// validateQuery0(safe, node, true, part);
+
+ // 3. Check query on remaining partition.
+ if (remainingPart != null) {
+ executeQuery(node, false, remainingPart);
+
+ // 4. Check query over two partitions - normal and LOST.
+ validateQuery0(safe, node, false, part, remainingPart);
+ }
+ }
+
+ /**
+ * Query validation routine.
+ *
+ * @param safe Safe flag.
+ * @param node Node.
+ * @param loc Local flag.
+ * @param parts Partitions.
+ */
+ private void validateQuery0(boolean safe, Ignite node, boolean loc, int... parts) {
+ if (safe) {
+ try {
+ executeQuery(node, loc, parts);
+
+ fail("Exception is not thrown.");
+ }
+ catch (Exception e) {
+ assertTrue(e.getMessage(), e.getMessage() != null &&
+ e.getMessage().contains("Failed to execute query because cache partition has been lost"));
+ }
+ }
+ else {
+ executeQuery(node, loc, parts);
+ }
+ }
+
+ /**
+ * Execute SQL query on a given node.
+ *
+ * @param parts Partitions.
+ * @param node Node.
+ * @param loc Local flag.
+ */
+ private static void executeQuery(Ignite node, boolean loc, int... parts) {
+ IgniteCache cache = node.cache(CACHE_NAME);
+
+ SqlFieldsQuery qry = new SqlFieldsQuery("SELECT * FROM Integer");
+
+ if (parts != null && parts.length != 0)
+ qry.setPartitions(parts);
+
+ if (loc)
+ qry.setLocal(true);
+
+ cache.query(qry).getAll();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0a23658e/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
index e26b211..d5ee0e9 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
@@ -158,7 +158,7 @@ public class IgniteCacheDistributedQueryCancelSelfTest extends GridCommonAbstrac
fail();
}
catch (Exception e) {
- assertTrue(e.getCause() instanceof CacheException);
+ assertTrue(e instanceof CacheException);
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0a23658e/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
index 3269887..ce38511 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
@@ -250,7 +250,8 @@ public class RetryCauseMessageSelfTest extends GridCommonAbstractTest {
personCache.query(qry).getAll();
}
catch (CacheException e) {
- assertTrue(e.getMessage().contains("Failed to reserve partitions for query (partition of PARTITIONED cache cannot be reserved) ["));
+ assertTrue(e.getMessage().contains("Failed to reserve partitions for query (partition of PARTITIONED " +
+ "cache is not found or not in OWNING state) "));
return;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0a23658e/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index 671db13..7f67b35 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -77,6 +77,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCheckClusterStateBefore
import org.apache.ignite.internal.processors.cache.IgniteCrossCachesJoinsQueryTest;
import org.apache.ignite.internal.processors.cache.IgniteDynamicSqlRestoreTest;
import org.apache.ignite.internal.processors.cache.IncorrectQueryEntityTest;
+import org.apache.ignite.internal.processors.cache.IndexingCachePartitionLossPolicySelfTest;
import org.apache.ignite.internal.processors.cache.QueryEntityCaseMismatchTest;
import org.apache.ignite.internal.processors.cache.SqlFieldsQuerySelfTest;
import org.apache.ignite.internal.processors.cache.authentication.SqlUserCommandSelfTest;
@@ -469,6 +470,9 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
suite.addTestSuite(H2StatementCacheSelfTest.class);
suite.addTestSuite(PreparedStatementExSelfTest.class);
+ // Partition loss.
+ suite.addTestSuite(IndexingCachePartitionLossPolicySelfTest.class);
+
return suite;
}
}
[33/50] [abbrv] ignite git commit: IGNITE-8158: Wrap the method
afterTestsStopped() call with try/catch. - Fixes #4464.
Posted by ag...@apache.org.
IGNITE-8158: Wrap the method afterTestsStopped() call with try/catch. - Fixes #4464.
Signed-off-by: Nikolay Izhikov <ni...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1894541c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1894541c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1894541c
Branch: refs/heads/ignite-5960
Commit: 1894541ce0301b7f4950fd30efcde15d0c4200b3
Parents: 1fd7247
Author: zzzadruga <zz...@gmail.com>
Authored: Wed Sep 5 14:52:04 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Wed Sep 5 14:52:55 2018 +0300
----------------------------------------------------------------------
.../testframework/junits/GridAbstractTest.java | 21 ++++++++++++++++----
1 file changed, 17 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1894541c/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index a1d6c25..ee0dfa4 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -1773,6 +1773,8 @@ public abstract class GridAbstractTest extends TestCase {
finally {
serializedObj.clear();
+ Exception err = null;
+
if (isLastTest()) {
info(">>> Stopping test class: " + testClassDescription() + " <<<");
@@ -1787,11 +1789,22 @@ public abstract class GridAbstractTest extends TestCase {
// Set reset flags, so counters will be reset on the next setUp.
counters.setReset(true);
- afterTestsStopped();
+ try {
+ afterTestsStopped();
+ }
+ catch (Exception e) {
+ err = e;
+ }
- if(isSafeTopology())
+ if (isSafeTopology()) {
stopAllGrids(false);
+ if (stopGridErr) {
+ err = new RuntimeException("Not all Ignite instances has been stopped. " +
+ "Please, see log for details.", err);
+ }
+ }
+
// Remove counters.
tests.remove(getClass());
@@ -1808,8 +1821,8 @@ public abstract class GridAbstractTest extends TestCase {
cleanReferences();
- if (isLastTest() && isSafeTopology() && stopGridErr)
- throw new RuntimeException("Not all Ignite instances has been stopped. Please, see log for details.");
+ if (err != null)
+ throw err;
}
}
[41/50] [abbrv] ignite git commit: IGNITE-9479 Fixed spontaneous
rebalance during cache start and improved logging level - Fixes #4691.
Posted by ag...@apache.org.
IGNITE-9479 Fixed spontaneous rebalance during cache start and improved logging level - Fixes #4691.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d5432c00
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d5432c00
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d5432c00
Branch: refs/heads/ignite-5960
Commit: d5432c005c378edaaa1f9a21a42241ecea15023e
Parents: 445d375
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Thu Sep 6 16:30:00 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Sep 6 16:30:00 2018 +0300
----------------------------------------------------------------------
.../dht/preloader/GridDhtForceKeysFuture.java | 36 ++--
.../dht/preloader/GridDhtPartitionDemander.java | 22 +--
.../dht/preloader/GridDhtPartitionSupplier.java | 4 +-
.../GridDhtPartitionsExchangeFuture.java | 10 +-
.../dht/preloader/GridDhtPreloader.java | 43 ++---
.../dht/IgniteCacheStartWithLoadTest.java | 165 +++++++++++++++++++
.../testsuites/IgniteCacheTestSuite7.java | 2 +
7 files changed, 213 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d5432c00/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java
index 52691f0..3b03958 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java
@@ -274,8 +274,8 @@ public final class GridDhtForceKeysFuture<K, V> extends GridCompoundFuture<Objec
assert !n.id().equals(loc.id());
- if (log.isDebugEnabled())
- log.debug("Sending force key request [cacheName=" + cctx.name() + "node=" + n.id() +
+ if (log.isTraceEnabled())
+ log.trace("Sending force key request [cacheName=" + cctx.name() + "node=" + n.id() +
", req=" + req + ']');
cctx.io().send(n, req, cctx.ioPolicy());
@@ -310,10 +310,10 @@ public final class GridDhtForceKeysFuture<K, V> extends GridCompoundFuture<Objec
try {
if (e != null && !e.isNewLocked()) {
- if (log.isDebugEnabled()) {
+ if (log.isTraceEnabled()) {
int part = cctx.affinity().partition(key);
- log.debug("Will not rebalance key (entry is not new) [cacheName=" + cctx.name() +
+ log.trace("Will not rebalance key (entry is not new) [cacheName=" + cctx.name() +
", key=" + key + ", part=" + part + ", locId=" + cctx.nodeId() + ']');
}
@@ -322,8 +322,8 @@ public final class GridDhtForceKeysFuture<K, V> extends GridCompoundFuture<Objec
}
}
catch (GridCacheEntryRemovedException ignore) {
- if (log.isDebugEnabled())
- log.debug("Received removed DHT entry for force keys request [entry=" + e +
+ if (log.isTraceEnabled())
+ log.trace("Received removed DHT entry for force keys request [entry=" + e +
", locId=" + cctx.nodeId() + ']');
}
@@ -333,8 +333,8 @@ public final class GridDhtForceKeysFuture<K, V> extends GridCompoundFuture<Objec
new ArrayList<>(F.view(top.owners(part, topVer), F.notIn(exc)));
if (owners.isEmpty() || (owners.contains(loc) && cctx.rebalanceEnabled())) {
- if (log.isDebugEnabled())
- log.debug("Will not rebalance key (local node is owner) [key=" + key + ", part=" + part +
+ if (log.isTraceEnabled())
+ log.trace("Will not rebalance key (local node is owner) [key=" + key + ", part=" + part +
"topVer=" + topVer + ", locId=" + cctx.nodeId() + ']');
// Key is already rebalanced.
@@ -344,8 +344,8 @@ public final class GridDhtForceKeysFuture<K, V> extends GridCompoundFuture<Objec
// Create partition.
GridDhtLocalPartition locPart = top.localPartition(part, topVer, false);
- if (log.isDebugEnabled())
- log.debug("Mapping local partition [loc=" + cctx.localNodeId() + ", topVer" + topVer +
+ if (log.isTraceEnabled())
+ log.trace("Mapping local partition [loc=" + cctx.localNodeId() + ", topVer" + topVer +
", part=" + locPart + ", owners=" + owners + ", allOwners=" + U.toShortString(top.owners(part)) + ']');
if (locPart == null)
@@ -362,8 +362,8 @@ public final class GridDhtForceKeysFuture<K, V> extends GridCompoundFuture<Objec
pick = F.first(F.view(owners, F.remoteNodes(loc.id())));
if (pick == null) {
- if (log.isDebugEnabled())
- log.debug("Will not rebalance key (no nodes to request from with rebalancing disabled) [key=" +
+ if (log.isTraceEnabled())
+ log.trace("Will not rebalance key (no nodes to request from with rebalancing disabled) [key=" +
key + ", part=" + part + ", locId=" + cctx.nodeId() + ']');
return mappings;
@@ -378,15 +378,15 @@ public final class GridDhtForceKeysFuture<K, V> extends GridCompoundFuture<Objec
mappedKeys.add(key);
- if (log.isDebugEnabled())
- log.debug("Will rebalance key from node [cacheName=" + cctx.name() + ", key=" + key + ", part=" +
+ if (log.isTraceEnabled())
+ log.trace("Will rebalance key from node [cacheName=" + cctx.name() + ", key=" + key + ", part=" +
part + ", node=" + pick.id() + ", locId=" + cctx.nodeId() + ']');
}
else if (locPart.state() != OWNING)
invalidParts.add(part);
else {
- if (log.isDebugEnabled())
- log.debug("Will not rebalance key (local partition is not MOVING) [cacheName=" + cctx.name() +
+ if (log.isTraceEnabled())
+ log.trace("Will not rebalance key (local partition is not MOVING) [cacheName=" + cctx.name() +
", key=" + key + ", part=" + locPart + ", locId=" + cctx.nodeId() + ']');
}
@@ -563,8 +563,8 @@ public final class GridDhtForceKeysFuture<K, V> extends GridCompoundFuture<Objec
return;
}
catch (GridCacheEntryRemovedException ignore) {
- if (log.isDebugEnabled())
- log.debug("Trying to rebalance removed entry (will ignore) [cacheName=" +
+ if (log.isTraceEnabled())
+ log.trace("Trying to rebalance removed entry (will ignore) [cacheName=" +
cctx.name() + ", entry=" + entry + ']');
}
finally {
http://git-wip-us.apache.org/repos/asf/ignite/blob/d5432c00/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index a0f82ea..851fcc9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -761,9 +761,9 @@ public class GridDhtPartitionDemander {
GridCacheEntryInfo entry = infos.next();
if (!preloadEntry(node, p, entry, topVer)) {
- if (log.isDebugEnabled())
- log.debug("Got entries for invalid partition during " +
- "preloading (will skip) [p=" + p + ", entry=" + entry + ']');
+ if (log.isTraceEnabled())
+ log.trace("Got entries for invalid partition during " +
+ "preloading (will skip) [p=" + p + ", entry=" + entry + ']');
break;
}
@@ -877,8 +877,8 @@ public class GridDhtPartitionDemander {
cached = cctx.cache().entryEx(entry.key());
- if (log.isDebugEnabled())
- log.debug("Rebalancing key [key=" + entry.key() + ", part=" + p + ", node=" + from.id() + ']');
+ if (log.isTraceEnabled())
+ log.trace("Rebalancing key [key=" + entry.key() + ", part=" + p + ", node=" + from.id() + ']');
if (preloadPred == null || preloadPred.apply(entry)) {
if (cached.initialValue(
@@ -905,17 +905,17 @@ public class GridDhtPartitionDemander {
else {
cached.touch(topVer); // Start tracking.
- if (log.isDebugEnabled())
- log.debug("Rebalancing entry is already in cache (will ignore) [key=" + cached.key() +
+ if (log.isTraceEnabled())
+ log.trace("Rebalancing entry is already in cache (will ignore) [key=" + cached.key() +
", part=" + p + ']');
}
}
- else if (log.isDebugEnabled())
- log.debug("Rebalance predicate evaluated to false for entry (will ignore): " + entry);
+ else if (log.isTraceEnabled())
+ log.trace("Rebalance predicate evaluated to false for entry (will ignore): " + entry);
}
catch (GridCacheEntryRemovedException ignored) {
- if (log.isDebugEnabled())
- log.debug("Entry has been concurrently removed while rebalancing (will ignore) [key=" +
+ if (log.isTraceEnabled())
+ log.trace("Entry has been concurrently removed while rebalancing (will ignore) [key=" +
cached.key() + ", part=" + p + ']');
}
catch (GridDhtInvalidPartitionException ignored) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/d5432c00/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 7ce4f7e..524d02d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -402,8 +402,8 @@ class GridDhtPartitionSupplier {
if (preloadPred == null || preloadPred.apply(info))
s.addEntry0(part, iter.historical(part), info, grp.shared(), grp.cacheObjectContext());
else {
- if (log.isDebugEnabled())
- log.debug("Rebalance predicate evaluated to false (will not send " +
+ if (log.isTraceEnabled())
+ log.trace("Rebalance predicate evaluated to false (will not send " +
"cache entry): " + info);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d5432c00/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 357f3d7..e66511c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1225,8 +1225,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
boolean distributed = true;
- // Do not perform distributed partition release in case of cluster activation or caches start.
- if (activateCluster() || hasCachesToStart())
+ // Do not perform distributed partition release in case of cluster activation.
+ if (activateCluster())
distributed = false;
// On first phase we wait for finishing all local tx updates, atomic updates and lock releases on all nodes.
@@ -2920,11 +2920,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
}
- // Don't validate partitions state in case of caches start.
- boolean skipValidation = hasCachesToStart();
-
- if (!skipValidation)
- validatePartitionsState();
+ validatePartitionsState();
if (firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) {
assert firstDiscoEvt instanceof DiscoveryCustomEvent;
http://git-wip-us.apache.org/repos/asf/ignite/blob/d5432c00/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 7cf55a3..f886767 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -313,7 +313,7 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter {
msg.partitions().addHistorical(p, part.initialUpdateCounter(), countersMap.updateCounter(p), partCnt);
}
else {
- Collection<ClusterNode> picked = pickOwners(p, topVer);
+ List<ClusterNode> picked = remoteOwners(p, topVer);
if (picked.isEmpty()) {
top.own(part);
@@ -330,7 +330,7 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter {
log.debug("Owning partition as there are no other owners: " + part);
}
else {
- ClusterNode n = F.rand(picked);
+ ClusterNode n = picked.get(0);
GridDhtPartitionDemandMessage msg = assignments.get(n);
@@ -359,42 +359,23 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter {
}
/**
- * Picks owners for specified partition {@code p} from affinity.
+ * Returns remote owners (excluding local node) for specified partition {@code p}.
*
* @param p Partition.
* @param topVer Topology version.
- * @return Picked owners.
+ * @return Nodes owning this partition.
*/
- private Collection<ClusterNode> pickOwners(int p, AffinityTopologyVersion topVer) {
- Collection<ClusterNode> affNodes = grp.affinity().cachedAffinity(topVer).get(p);
-
- int affCnt = affNodes.size();
-
- Collection<ClusterNode> rmts = remoteOwners(p, topVer);
+ private List<ClusterNode> remoteOwners(int p, AffinityTopologyVersion topVer) {
+ List<ClusterNode> owners = grp.topology().owners(p, topVer);
- int rmtCnt = rmts.size();
+ List<ClusterNode> res = new ArrayList<>(owners.size());
- if (rmtCnt <= affCnt)
- return rmts;
-
- List<ClusterNode> sorted = new ArrayList<>(rmts);
-
- // Sort in descending order, so nodes with higher order will be first.
- Collections.sort(sorted, CU.nodeComparator(false));
-
- // Pick newest nodes.
- return sorted.subList(0, affCnt);
- }
+ for (ClusterNode owner : owners) {
+ if (!owner.id().equals(ctx.localNodeId()))
+ res.add(owner);
+ }
- /**
- * Returns remote owners (excluding local node) for specified partition {@code p}.
- *
- * @param p Partition.
- * @param topVer Topology version.
- * @return Nodes owning this partition.
- */
- private Collection<ClusterNode> remoteOwners(int p, AffinityTopologyVersion topVer) {
- return F.view(grp.topology().owners(p, topVer), F.remoteNodes(ctx.localNodeId()));
+ return res;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/d5432c00/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheStartWithLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheStartWithLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheStartWithLoadTest.java
new file mode 100644
index 0000000..acccc5b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheStartWithLoadTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class IgniteCacheStartWithLoadTest extends GridCommonAbstractTest {
+ /** */
+ static final String CACHE_NAME = "tx_repl";
+
+ @Override
+ protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setConsistentId(igniteInstanceName);
+
+ CacheConfiguration ccfg = new CacheConfiguration().setName(CACHE_NAME)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setCacheMode(CacheMode.REPLICATED)
+ .setDataRegionName("ds")
+ .setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+ .setDataRegionConfigurations(
+ new DataRegionConfiguration()
+ .setName("ds")
+ .setPersistenceEnabled(true)
+ .setMaxSize(1024 * 1024 * 1024)
+ );
+
+ cfg.setDataStorageConfiguration(dsCfg);
+ cfg.setCacheConfiguration(ccfg);
+
+ return cfg;
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testNoRebalanceDuringCacheStart() throws Exception {
+ IgniteEx crd = (IgniteEx)startGrids(4);
+
+ crd.cluster().active(true);
+
+ AtomicBoolean txLoadStop = new AtomicBoolean();
+
+ AtomicInteger txLoaderNo = new AtomicInteger(0);
+
+ IgniteInternalFuture txLoadFuture = GridTestUtils.runMultiThreadedAsync(() -> {
+ Ignite node = grid(txLoaderNo.getAndIncrement());
+ IgniteCache<Object, Object> cache = node.cache(CACHE_NAME);
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ final int keys = 5;
+ final int keysSpace = 10_000;
+
+ while (!txLoadStop.get()) {
+ try (Transaction tx = node.transactions().txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+ for (int it = 0; it < keys; it++) {
+ int key = rnd.nextInt(keysSpace);
+ byte[] value = new byte[2048];
+
+ cache.put(key, value);
+ }
+ tx.commit();
+
+ U.sleep(10);
+ }
+ catch (Throwable t) {
+ log.warning("Unexpected exception during tx load.", t);
+ }
+ }
+ }, 4, "tx-loader");
+
+ AtomicBoolean hasRebalance = new AtomicBoolean();
+
+ AtomicBoolean cacheRestartStop = new AtomicBoolean();
+
+ IgniteInternalFuture cacheRestartFuture = GridTestUtils.runAsync(() -> {
+ Ignite node = grid(0);
+
+ final String tmpCacheName = "tmp";
+
+ while (!cacheRestartStop.get()) {
+ try {
+ node.getOrCreateCache(tmpCacheName);
+
+ boolean hasMoving = false;
+
+ for (int i = 0; i < 4; i++) {
+ hasMoving |= grid(i).cachex(CACHE_NAME).context().topology().hasMovingPartitions();
+ }
+
+ if (hasMoving) {
+ log.error("Cache restarter has been stopped because rebalance is triggered for stable caches.");
+
+ hasRebalance.set(true);
+
+ return;
+ }
+
+ node.destroyCache(tmpCacheName);
+
+ U.sleep(10_000);
+ }
+ catch (Throwable t) {
+ log.warning("Unexpected exception during caches restart.", t);
+ }
+ }
+ });
+
+ U.sleep(60_000);
+
+ cacheRestartStop.set(true);
+ txLoadStop.set(true);
+
+ cacheRestartFuture.get();
+ txLoadFuture.get();
+
+ Assert.assertFalse(hasRebalance.get());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 5 * 60 * 1000;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d5432c00/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
index 3a35950..0a2f86e 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
@@ -35,6 +35,7 @@ import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheRentingStateRepairTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheDataLossOnPartitionMoveTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheStartWithLoadTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingTest;
import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMultinodeMixedRegionsTest;
@@ -65,6 +66,7 @@ public class IgniteCacheTestSuite7 extends TestSuite {
TestSuite suite = new TestSuite("IgniteCache With Persistence Test Suite");
suite.addTestSuite(CheckpointBufferDeadlockTest.class);
+ suite.addTestSuite(IgniteCacheStartWithLoadTest.class);
suite.addTestSuite(AuthenticationConfigurationClusterTest.class);
suite.addTestSuite(AuthenticationProcessorSelfTest.class);
[49/50] [abbrv] ignite git commit: IGNITE-8149: MVCC: correct
processing of cache size. This closes #4654.
Posted by ag...@apache.org.
IGNITE-8149: MVCC: correct processing of cache size. This closes #4654.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/00e935d3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/00e935d3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/00e935d3
Branch: refs/heads/ignite-5960
Commit: 00e935d33cd852cdc2a460f24ca0f0136ac5c59e
Parents: 40432b0
Author: ipavlukhin <vo...@gmail.com>
Authored: Fri Sep 7 16:00:02 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Sep 7 16:00:02 2018 +0300
----------------------------------------------------------------------
.../communication/GridIoMessageFactory.java | 4 +-
.../processors/cache/GridCacheMapEntry.java | 12 +
.../cache/IgniteCacheOffheapManager.java | 8 +
.../cache/IgniteCacheOffheapManagerImpl.java | 54 +-
.../GridDistributedTxRemoteAdapter.java | 51 +-
.../dht/GridDhtPartitionsUpdateCountersMap.java | 119 -----
.../dht/GridDhtTxAbstractEnlistFuture.java | 2 +-
.../distributed/dht/GridDhtTxFinishFuture.java | 6 +-
.../distributed/dht/GridDhtTxFinishRequest.java | 14 +-
.../distributed/dht/GridDhtTxLocalAdapter.java | 51 +-
.../cache/distributed/dht/GridDhtTxRemote.java | 12 -
.../dht/PartitionUpdateCounters.java | 123 +++++
.../persistence/GridCacheOffheapManager.java | 13 +
.../cache/transactions/IgniteInternalTx.java | 8 +-
.../cache/transactions/IgniteTxAdapter.java | 57 ++-
.../cache/transactions/IgniteTxHandler.java | 2 +-
.../transactions/IgniteTxLocalAdapter.java | 104 ++--
.../cache/transactions/IgniteTxLocalEx.java | 8 +
.../cache/transactions/IgniteTxLocalState.java | 16 +
.../transactions/IgniteTxLocalStateAdapter.java | 26 +
.../cache/transactions/IgniteTxRemoteEx.java | 12 -
.../cache/transactions/TxCounters.java | 82 ++++
.../cache/mvcc/CacheMvccSizeTest.java | 488 +++++++++++++++++++
.../testsuites/IgniteCacheMvccSqlTestSuite.java | 5 +-
24 files changed, 990 insertions(+), 287 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 2970e71..14f0548 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -70,7 +70,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffini
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLockRequest;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLockResponse;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionsUpdateCountersMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRequest;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishResponse;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxOnePhaseCommitAckRequest;
@@ -80,6 +79,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxQuer
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxQueryEnlistResponse;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxQueryFirstEnlistRequest;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtUnlockRequest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.PartitionUpdateCounters;
import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse;
import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicNearResponse;
import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicSingleUpdateRequest;
@@ -1062,7 +1062,7 @@ public class GridIoMessageFactory implements MessageFactory {
break;
case 157:
- msg = new GridDhtPartitionsUpdateCountersMap();
+ msg = new PartitionUpdateCounters();
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 8c2b939..714d4a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -1122,6 +1122,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (tx.local())
updateCntr = nextMvccPartitionCounter();
+ if (res.resultType() == ResultType.PREV_NULL)
+ tx.txCounters(true).accumulateSizeDelta(cctx.cacheId(), partition(), 1);
+
if (cctx.group().persistenceEnabled() && cctx.group().walEnabled())
logPtr = cctx.shared().wal().log(new DataRecord(new DataEntry(
cctx.cacheId(),
@@ -1217,6 +1220,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (tx.local())
updateCntr = nextMvccPartitionCounter();
+ if (res.resultType() == ResultType.PREV_NOT_NULL)
+ tx.txCounters(true).accumulateSizeDelta(cctx.cacheId(), partition(), -1);
+
if (cctx.group().persistenceEnabled() && cctx.group().walEnabled())
logPtr = logTxUpdate(tx, null, 0, updateCntr);
@@ -4994,6 +5000,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (updateCntr != null && updateCntr != 0)
updateCntr0 = updateCntr;
+ if (res.resultType() == ResultType.PREV_NOT_NULL)
+ tx.txCounters(true).accumulateSizeDelta(cctx.cacheId(), entry.partition(), -1);
+
if (cctx.group().persistenceEnabled() && cctx.group().walEnabled())
logPtr = cctx.shared().wal().log(new DataRecord(new DataEntry(
cctx.cacheId(),
@@ -5281,6 +5290,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
updateCntr0 = tx.local() ? entry.nextMvccPartitionCounter() : updateCntr;
+ if (res.resultType() == ResultType.PREV_NULL)
+ tx.txCounters(true).accumulateSizeDelta(cctx.cacheId(), entry.partition(), 1);
+
if (cctx.group().persistenceEnabled() && cctx.group().walEnabled())
logPtr = cctx.shared().wal().log(new DataRecord(new DataEntry(
cctx.cacheId(),
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
index a021394..fdf42fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
@@ -620,6 +620,14 @@ public interface IgniteCacheOffheapManager {
long fullSize();
/**
+ * Updates size metric for particular cache.
+ *
+ * @param cacheId Cache ID.
+ * @param delta Size delta.
+ */
+ void updateSize(int cacheId, long delta);
+
+ /**
* @return Update counter.
*/
long updateCounter();
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 13ad7e2..8811006 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -1450,36 +1450,14 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
* @param cacheId Cache ID.
*/
void incrementSize(int cacheId) {
- storageSize.incrementAndGet();
-
- if (grp.sharedGroup()) {
- AtomicLong size = cacheSizes.get(cacheId);
-
- if (size == null) {
- AtomicLong old = cacheSizes.putIfAbsent(cacheId, size = new AtomicLong());
-
- if (old != null)
- size = old;
- }
-
- size.incrementAndGet();
- }
+ updateSize(cacheId, 1);
}
/**
* @param cacheId Cache ID.
*/
void decrementSize(int cacheId) {
- storageSize.decrementAndGet();
-
- if (grp.sharedGroup()) {
- AtomicLong size = cacheSizes.get(cacheId);
-
- if (size == null)
- return;
-
- size.decrementAndGet();
- }
+ updateSize(cacheId, -1);
}
/** {@inheritDoc} */
@@ -1517,6 +1495,24 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
}
/** {@inheritDoc} */
+ @Override public void updateSize(int cacheId, long delta) {
+ storageSize.addAndGet(delta);
+
+ if (grp.sharedGroup()) {
+ AtomicLong size = cacheSizes.get(cacheId);
+
+ if (size == null) {
+ AtomicLong old = cacheSizes.putIfAbsent(cacheId, size = new AtomicLong());
+
+ if (old != null)
+ size = old;
+ }
+
+ size.addAndGet(delta);
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public long nextUpdateCounter() {
return cntr.incrementAndGet();
}
@@ -1953,8 +1949,6 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
assert !old;
- incrementSize(cctx.cacheId());
-
GridCacheQueryManager qryMgr = cctx.queries();
if (qryMgr.enabled())
@@ -2284,11 +2278,13 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
rowStore.removeRow(row.link());
- decrementSize(cctx.cacheId());
-
if (first)
first = false;
}
+
+ // first == true means there were no row versions
+ if (!first)
+ decrementSize(cctx.cacheId());
}
/** {@inheritDoc} */
@@ -2318,8 +2314,6 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
rowStore.removeRow(cleanupRow.link());
- decrementSize(cctx.cacheId());
-
res++;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index dd6ea48..8e96ae2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -51,7 +51,8 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.GridCacheUpdateTxResult;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionsUpdateCountersMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.PartitionUpdateCounters;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry;
import org.apache.ignite.internal.processors.cache.persistence.StorageException;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
@@ -61,6 +62,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxRemoteEx;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxRemoteState;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxState;
+import org.apache.ignite.internal.processors.cache.transactions.TxCounters;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
@@ -807,7 +809,7 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
}
}
- updateLocalCounters();
+ applyTxCounters();
if (!near() && !F.isEmpty(dataEntries) && cctx.wal() != null) {
// Set new update counters for data entries received from persisted tx entries.
@@ -845,29 +847,34 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
}
}
- /**
- * Applies update counters to the local partitions.
- */
- private void updateLocalCounters() {
- Map<Integer, GridDhtPartitionsUpdateCountersMap> updCntrsMap = updateCountersMap();
+ /** {@inheritDoc} */
+ @Override protected void applyTxCounters() {
+ super.applyTxCounters();
+
+ TxCounters txCntrs = txCounters(false);
- if (F.isEmpty(updCntrsMap))
+ if (txCntrs == null)
return;
- for (Map.Entry<Integer, GridDhtPartitionsUpdateCountersMap> entry : updCntrsMap.entrySet()) {
- GridCacheContext cacheCtx = cctx.cacheContext(entry.getKey());
+ Map<Integer, PartitionUpdateCounters> updCntrs = txCntrs.updateCounters();
- GridDhtPartitionsUpdateCountersMap cacheUpdCntrs = entry.getValue();
+ for (Map.Entry<Integer, PartitionUpdateCounters> entry : updCntrs.entrySet()) {
+ int cacheId = entry.getKey();
- assert cacheUpdCntrs != null && !F.isEmpty(cacheUpdCntrs.updateCounters());
+ GridDhtPartitionTopology top = cctx.cacheContext(cacheId).topology();
- for (Map.Entry<Integer, Long> e : cacheUpdCntrs.updateCounters().entrySet()) {
- Long updCntr = e.getValue();
- GridDhtLocalPartition part = cacheCtx.topology().localPartition(e.getKey());
+ Map<Integer, Long> cacheUpdCntrs = entry.getValue().updateCounters();
- assert part != null && updCntr != null && updCntr > 0;
+ assert cacheUpdCntrs != null;
- part.updateCounter(updCntr);
+ for (Map.Entry<Integer, Long> e : cacheUpdCntrs.entrySet()) {
+ long updCntr = e.getValue();
+
+ GridDhtLocalPartition dhtPart = top.localPartition(e.getKey());
+
+ assert dhtPart != null && updCntr > 0;
+
+ dhtPart.updateCounter(updCntr);
}
}
}
@@ -1023,16 +1030,6 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
}
/** {@inheritDoc} */
- @Override public void updateCountersMap(Map<Integer, GridDhtPartitionsUpdateCountersMap> updCntrsMap) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public Map<Integer, GridDhtPartitionsUpdateCountersMap> updateCountersMap() {
- return null;
- }
-
- /** {@inheritDoc} */
@Override public String toString() {
return GridToStringBuilder.toString(GridDistributedTxRemoteAdapter.class, this, "super", super.toString());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsUpdateCountersMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsUpdateCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsUpdateCountersMap.java
deleted file mode 100644
index 7a345d1..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsUpdateCountersMap.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.distributed.dht;
-
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.internal.GridDirectMap;
-import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
-import org.apache.ignite.plugin.extensions.communication.MessageReader;
-import org.apache.ignite.plugin.extensions.communication.MessageWriter;
-
-/**
- * Partitions update counters message.
- */
-public class GridDhtPartitionsUpdateCountersMap implements Message {
- /** */
- private static final long serialVersionUID = -4599730112233297219L;
-
- /** Map of update counters made by this tx. Mapping: partId -> updCntr. */
- @GridDirectMap(keyType = Integer.class, valueType = Long.class)
- private Map<Integer, Long> updCntrs;
-
- /**
- *
- */
- public GridDhtPartitionsUpdateCountersMap() {
- updCntrs = new HashMap<>();
- }
-
- /**
- * @return Update counters.
- */
- public Map<Integer, Long> updateCounters() {
- return updCntrs;
- }
-
- /**
- * @param updCntrs Update counters.
- */
- public void updateCounters(Map<Integer, Long> updCntrs) {
- this.updCntrs = updCntrs;
- }
-
- /** {@inheritDoc} */
- @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
- writer.setBuffer(buf);
-
- if (!writer.isHeaderWritten()) {
- if (!writer.writeHeader(directType(), fieldsCount()))
- return false;
-
- writer.onHeaderWritten();
- }
-
- switch (writer.state()) {
- case 0:
- if (!writer.writeMap("updCntrs", updCntrs, MessageCollectionItemType.INT, MessageCollectionItemType.LONG))
- return false;
-
- writer.incrementState();
-
- }
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
- reader.setBuffer(buf);
-
- if (!reader.beforeMessageRead())
- return false;
-
- switch (reader.state()) {
- case 0:
- updCntrs = reader.readMap("updCntrs", MessageCollectionItemType.INT, MessageCollectionItemType.LONG, false);
-
- if (!reader.isLastRead())
- return false;
-
- reader.incrementState();
-
- }
-
- return reader.afterMessageRead(GridDhtPartitionsUpdateCountersMap.class);
- }
-
- /** {@inheritDoc} */
- @Override public short directType() {
- return 157;
- }
-
- /** {@inheritDoc} */
- @Override public byte fieldsCount() {
- return 1;
- }
-
- /** {@inheritDoc} */
- @Override public void onAckReceived() {
- // No-op.
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java
index bb11df5..a3471c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java
@@ -593,7 +593,7 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
int part = cctx.affinity().partition(key);
- tx.addPartitionCountersMapping(cacheId, part);
+ tx.touchPartition(cacheId, part);
if (F.isEmpty(backups))
return;
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
index c0a3845..4c72e6f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
@@ -457,10 +457,10 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCacheCompoundIdentity
for (IgniteTxEntry e : dhtMapping.entries())
updCntrs.add(e.updateCounter());
- Map<Integer, GridDhtPartitionsUpdateCountersMap> updCntrsMap = null;
+ Map<Integer, PartitionUpdateCounters> updCntrsForNode = null;
if (dhtMapping.queryUpdate() && commit)
- updCntrsMap = tx.updateCountersForNode(n);
+ updCntrsForNode = tx.filterUpdateCountersForBackupNode(n);
GridDhtTxFinishRequest req = new GridDhtTxFinishRequest(
tx.nearNodeId(),
@@ -489,7 +489,7 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCacheCompoundIdentity
false,
false,
mvccSnapshot,
- updCntrsMap);
+ updCntrsForNode);
req.writeVersion(tx.writeVersion() != null ? tx.writeVersion() : tx.xidVersion());
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
index b1908ff..8e9ece6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
@@ -73,9 +73,10 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
/** */
private MvccSnapshot mvccSnapshot;
- /** Map of update counters made by this tx. Mapping: cacheId -> partId -> updCntr. */
- @GridDirectMap(keyType = Integer.class, valueType = GridDhtPartitionsUpdateCountersMap.class)
- private Map<Integer, GridDhtPartitionsUpdateCountersMap> updCntrs;
+ /** */
+ @GridDirectMap(keyType = Integer.class, valueType = PartitionUpdateCounters.class)
+ private Map<Integer, PartitionUpdateCounters> updCntrs;
+
/**
* Empty constructor required for {@link Externalizable}.
*/
@@ -220,7 +221,7 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
boolean retVal,
boolean waitRemoteTxs,
MvccSnapshot mvccSnapshot,
- Map<Integer, GridDhtPartitionsUpdateCountersMap> updCntrs
+ Map<Integer, PartitionUpdateCounters> updCntrs
) {
this(nearNodeId,
futId,
@@ -362,10 +363,11 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
public void needReturnValue(boolean retVal) {
setFlag(retVal, NEED_RETURN_VALUE_FLAG_MASK);
}
+
/**
- * @return Partition update counters map.
+ * @return Partition counters update deferred until transaction commit.
*/
- public Map<Integer, GridDhtPartitionsUpdateCountersMap> updateCountersMap() {
+ public Map<Integer, PartitionUpdateCounters> updateCounters() {
return updCntrs;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
index 11b46a0..613f160 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.distributed.dht;
import java.io.Externalizable;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -31,6 +32,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheAffinityManager;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
@@ -42,6 +44,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPr
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter;
+import org.apache.ignite.internal.processors.cache.transactions.TxCounters;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.F0;
import org.apache.ignite.internal.util.GridLeanMap;
@@ -907,7 +910,7 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
*
* @return Current lock future or null if it's safe to roll back.
*/
- public @Nullable IgniteInternalFuture<?> tryRollbackAsync() {
+ @Nullable public IgniteInternalFuture<?> tryRollbackAsync() {
while (true) {
final IgniteInternalFuture fut = lockFut;
@@ -937,6 +940,52 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
return prepFut;
}
+ /**
+ * @param node Backup node.
+ * @return Partition counters map for the given backup node.
+ */
+ public Map<Integer, PartitionUpdateCounters> filterUpdateCountersForBackupNode(ClusterNode node) {
+ TxCounters txCntrs = txCounters(false);
+
+ if (txCntrs == null)
+ return null;
+
+ Map<Integer, PartitionUpdateCounters> updCntrs = txCntrs.updateCounters();
+
+ Map<Integer, PartitionUpdateCounters> res = new HashMap<>();
+
+ AffinityTopologyVersion top = topologyVersionSnapshot();
+
+ for (Map.Entry<Integer, PartitionUpdateCounters> entry : updCntrs.entrySet()) {
+ Integer cacheId = entry.getKey();
+
+ Map<Integer, Long> partsCntrs = entry.getValue().updateCounters();
+
+ assert !F.isEmpty(partsCntrs);
+
+ GridCacheAffinityManager affinity = cctx.cacheContext(cacheId).affinity();
+
+ Map<Integer, Long> resCntrs = new HashMap<>(partsCntrs.size());
+
+ for (Map.Entry<Integer, Long> e : partsCntrs.entrySet()) {
+ Integer p = e.getKey();
+
+ Long cntr = e.getValue();
+
+ if (affinity.backupByPartition(node, p, top)) {
+ assert cntr != null && cntr > 0 : cntr;
+
+ resCntrs.put(p, cntr);
+ }
+ }
+
+ if (!resCntrs.isEmpty())
+ res.put(cacheId, new PartitionUpdateCounters(resCntrs));
+ }
+
+ return res;
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return GridToStringBuilder.toString(GridDhtTxLocalAdapter.class, this, "nearNodes", nearMap.keySet(),
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
index 08ecf28..9a1763b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
@@ -79,8 +79,6 @@ public class GridDhtTxRemote extends GridDistributedTxRemoteAdapter {
/** Store write through flag. */
private boolean storeWriteThrough;
- /** Map of update counters made by this tx. Mapping: cacheId -> partId -> updCntr. */
- private Map<Integer, GridDhtPartitionsUpdateCountersMap> updCntrs;
/**
* Empty constructor required for {@link Externalizable}.
*/
@@ -503,16 +501,6 @@ public class GridDhtTxRemote extends GridDistributedTxRemoteAdapter {
}
/** {@inheritDoc} */
- @Override public void updateCountersMap(Map<Integer, GridDhtPartitionsUpdateCountersMap> updCntrsMap) {
- this.updCntrs = updCntrsMap;
- }
-
- /** {@inheritDoc} */
- @Override public Map<Integer, GridDhtPartitionsUpdateCountersMap> updateCountersMap() {
- return updCntrs;
- }
-
- /** {@inheritDoc} */
@Override public String toString() {
return GridToStringBuilder.toString(GridDhtTxRemote.class, this, "super", super.toString());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionUpdateCounters.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionUpdateCounters.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionUpdateCounters.java
new file mode 100644
index 0000000..5b1eccd
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionUpdateCounters.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+import org.apache.ignite.internal.GridDirectMap;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ * Partition update counters message.
+ */
+public class PartitionUpdateCounters implements Message {
+ /** */
+ private static final long serialVersionUID = 193442457510062844L;
+
+ /** Map of update counters made by this tx. Mapping: partId -> updCntr. */
+ @GridDirectMap(keyType = Integer.class, valueType = Long.class)
+ private Map<Integer, Long> updCntrs;
+
+ /** */
+ public PartitionUpdateCounters() {
+ // No-op.
+ }
+
+ /**
+ * @param updCntrs Update counters map.
+ */
+ public PartitionUpdateCounters(Map<Integer, Long> updCntrs) {
+ this.updCntrs = updCntrs;
+ }
+
+ /**
+ * @return Update counters.
+ */
+ public Map<Integer, Long> updateCounters() {
+ return updCntrs;
+ }
+
+ /**
+ * @param updCntrs Update counters.
+ */
+ public void updateCounters(Map<Integer, Long> updCntrs) {
+ this.updCntrs = updCntrs;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeMap("updCntrs", updCntrs, MessageCollectionItemType.INT, MessageCollectionItemType.LONG))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ updCntrs = reader.readMap("updCntrs", MessageCollectionItemType.INT, MessageCollectionItemType.LONG, false);
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(PartitionUpdateCounters.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return 157;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 1;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 199efcb..04476ad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -1563,6 +1563,19 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
}
/** {@inheritDoc} */
+ @Override public void updateSize(int cacheId, long delta) {
+ try {
+ CacheDataStore delegate0 = init0(false);
+
+ if (delegate0 != null)
+ delegate0.updateSize(cacheId, delta);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public long updateCounter() {
try {
CacheDataStore delegate0 = init0(true);
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
index 4acf078..05ebe5d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
@@ -650,4 +650,10 @@ public interface IgniteInternalTx {
* @return Mvcc snapshot.
*/
public MvccSnapshot mvccSnapshot();
-}
\ No newline at end of file
+
+ /**
+ * @return Transaction counters.
+ * @param createIfAbsent {@code True} if non-null instance is needed.
+ */
+ @Nullable public TxCounters txCounters(boolean createIfAbsent);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index ee5a58e..c6d1991 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -34,6 +34,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import javax.cache.expiry.ExpiryPolicy;
@@ -62,10 +63,12 @@ import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.GridCacheReturn;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
import org.apache.ignite.internal.processors.cache.mvcc.txlog.TxState;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
import org.apache.ignite.internal.processors.cache.store.CacheStoreManager;
import org.apache.ignite.internal.processors.cache.version.GridCacheLazyPlainVersionedEntry;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -136,6 +139,10 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
private static final AtomicReferenceFieldUpdater<IgniteTxAdapter, FinalizationStatus> FINALIZING_UPD =
AtomicReferenceFieldUpdater.newUpdater(IgniteTxAdapter.class, FinalizationStatus.class, "finalizing");
+ /** */
+ private static final AtomicReferenceFieldUpdater<IgniteTxAdapter, TxCounters> TX_COUNTERS_UPD =
+ AtomicReferenceFieldUpdater.newUpdater(IgniteTxAdapter.class, TxCounters.class, "txCounters");
+
/** Logger. */
protected static IgniteLogger log;
@@ -275,6 +282,9 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
@GridToStringExclude
private volatile IgniteInternalFuture rollbackFut;
+ /** */
+ private volatile TxCounters txCounters = new TxCounters();
+
/**
* Empty constructor required for {@link Externalizable}.
*/
@@ -2015,6 +2025,46 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
public abstract void addActiveCache(GridCacheContext cacheCtx, boolean recovery) throws IgniteCheckedException;
/** {@inheritDoc} */
+ @Nullable @Override public TxCounters txCounters(boolean createIfAbsent) {
+ if (createIfAbsent && txCounters == null)
+ TX_COUNTERS_UPD.compareAndSet(this, null, new TxCounters());
+
+ return txCounters;
+ }
+
+ /**
+ * Make counters accumulated during transaction visible outside of transaciton.
+ */
+ protected void applyTxCounters() {
+ TxCounters txCntrs = txCounters(false);
+
+ if (txCntrs == null)
+ return;
+
+ Map<Integer, ? extends Map<Integer, AtomicLong>> sizeDeltas = txCntrs.sizeDeltas();
+
+ for (Map.Entry<Integer, ? extends Map<Integer, AtomicLong>> entry : sizeDeltas.entrySet()) {
+ Integer cacheId = entry.getKey();
+ Map<Integer, AtomicLong> partDeltas = entry.getValue();
+
+ assert !F.isEmpty(partDeltas);
+
+ GridDhtPartitionTopology top = cctx.cacheContext(cacheId).topology();
+
+ for (Map.Entry<Integer, AtomicLong> e : partDeltas.entrySet()) {
+ Integer p = e.getKey();
+ long delta = e.getValue().get();
+
+ GridDhtLocalPartition dhtPart = top.localPartition(p);
+
+ assert dhtPart != null;
+
+ dhtPart.dataStore().updateSize(cacheId, delta);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return GridToStringBuilder.toString(IgniteTxAdapter.class, this,
"duration", (U.currentTimeMillis() - startTime) + "ms",
@@ -2287,6 +2337,11 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
}
/** {@inheritDoc} */
+ @Nullable @Override public TxCounters txCounters(boolean createIfAbsent) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Override public IgniteTxState txState() {
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 7541b43..32f4dd4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -1391,7 +1391,7 @@ public class IgniteTxHandler {
tx.setPartitionUpdateCounters(
req.partUpdateCounters() != null ? req.partUpdateCounters().array() : null);
- tx.updateCountersMap(req.updateCountersMap());
+ tx.txCounters(true).updateCounters(req.updateCounters());
tx.commitRemoteTx();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index b86273f..bfe67ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -24,7 +24,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import javax.cache.expiry.Duration;
@@ -32,7 +31,6 @@ import javax.cache.expiry.ExpiryPolicy;
import javax.cache.processor.EntryProcessor;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.IgniteInternalFuture;
@@ -59,7 +57,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheUpdateTxResult;
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionsUpdateCountersMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.PartitionUpdateCounters;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
import org.apache.ignite.internal.processors.cache.persistence.StorageException;
@@ -162,9 +160,6 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
/** */
private GridLongList mvccWaitTxs;
- /** Update counters map */
- private Map<Integer, Map<Integer, Long>> updCntrs;
-
/** */
private volatile boolean qryEnlisted;
@@ -918,7 +913,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
}
}
- updateLocalPartitionCounters();
+ applyTxCounters();
if (ptr != null && !cctx.tm().logTxRecords())
cctx.wal().flush(ptr, false);
@@ -1563,6 +1558,11 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
}
/** {@inheritDoc} */
+ @Override public void touchPartition(int cacheId, int partId) {
+ txState.touchPartition(cacheId, partId);
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return GridToStringBuilder.toString(IgniteTxLocalAdapter.class, this, "super", super.toString(),
"size", allEntries().size());
@@ -1644,74 +1644,28 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
}
/**
- * @return Partition counters map for the given backup node.
+ * Merges mvcc update counters to the partition update counters. For mvcc transactions we update partitions
+ * counters only on commit phase.
*/
- public Map<Integer, GridDhtPartitionsUpdateCountersMap> updateCountersForNode(ClusterNode node) {
- if (F.isEmpty(updCntrs))
+ private Map<Integer, PartitionUpdateCounters> applyAndCollectLocalUpdateCounters() {
+ if (F.isEmpty(txState.touchedPartitions()))
return null;
- Map<Integer, GridDhtPartitionsUpdateCountersMap> res = new HashMap<>();
+ HashMap<Integer, PartitionUpdateCounters> updCntrs = new HashMap<>();
- for (Map.Entry<Integer, Map<Integer, Long>> entry : updCntrs.entrySet()) {
- Map<Integer, Long> partsCntrs = entry.getValue();
+ for (Map.Entry<Integer, Set<Integer>> entry : txState.touchedPartitions().entrySet()) {
+ Integer cacheId = entry.getKey();
- assert !F.isEmpty(partsCntrs);
+ Set<Integer> parts = entry.getValue();
- GridCacheContext ctx0 = cctx.cacheContext(entry.getKey());
+ assert !F.isEmpty(parts);
- GridDhtPartitionsUpdateCountersMap resBackupCntrs = new GridDhtPartitionsUpdateCountersMap();
+ GridCacheContext ctx0 = cctx.cacheContext(cacheId);
- for (Map.Entry<Integer, Long> e : partsCntrs.entrySet()) {
- Long cntr = partsCntrs.get(e.getKey());
+ Map<Integer, Long> partCntrs = new HashMap<>(parts.size());
- if (ctx0.affinity().backupByPartition(node, e.getKey(), topologyVersionSnapshot())) {
- assert cntr != null && cntr > 0 : cntr;
-
- resBackupCntrs.updateCounters().put(e.getKey(), cntr);
- }
- }
-
- if (!resBackupCntrs.updateCounters().isEmpty())
- res.put(entry.getKey(), resBackupCntrs);
- }
-
- return res;
- }
-
- /**
- * @param cacheId Cache id.
- * @param part Partition id.
- */
- @SuppressWarnings("Java8MapApi")
- public void addPartitionCountersMapping(Integer cacheId, Integer part) {
- if (updCntrs == null)
- updCntrs = new ConcurrentHashMap<>();
-
- Map<Integer, Long> partUpdCntrs = updCntrs.get(cacheId);
-
- if (partUpdCntrs == null)
- updCntrs.put(cacheId, partUpdCntrs = new ConcurrentHashMap<>());
-
- partUpdCntrs.put(part, 0L);
- }
-
- /**
- * Merges mvcc update counters to the partition update counters. For mvcc transactions we update partitions
- * counters only on commit phase.
- */
- private void updateLocalPartitionCounters() {
- if (F.isEmpty(updCntrs))
- return;
-
- for (Map.Entry<Integer, Map<Integer, Long>> entry : updCntrs.entrySet()) {
- Map<Integer, Long> partsCntrs = entry.getValue();
-
- assert !F.isEmpty(partsCntrs);
-
- GridCacheContext ctx0 = cctx.cacheContext(entry.getKey());
-
- for (Map.Entry<Integer, Long> e : partsCntrs.entrySet()) {
- GridDhtLocalPartition dhtPart = ctx0.topology().localPartition(e.getKey());
+ for (Integer p : parts) {
+ GridDhtLocalPartition dhtPart = ctx0.topology().localPartition(p);
assert dhtPart != null;
@@ -1719,11 +1673,13 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
dhtPart.updateCounter(cntr);
- Long prev = partsCntrs.put(e.getKey(), cntr);
-
- assert prev == 0L : prev;
+ partCntrs.put(p, cntr);
}
+
+ updCntrs.put(cacheId, new PartitionUpdateCounters(partCntrs));
}
+
+ return updCntrs;
}
/**
@@ -1747,6 +1703,16 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
}
}
+ /** {@inheritDoc} */
+ @Override protected void applyTxCounters() {
+ super.applyTxCounters();
+
+ Map<Integer, PartitionUpdateCounters> updCntrs = applyAndCollectLocalUpdateCounters();
+
+ // remember counters for subsequent sending to backups
+ txCounters(true).updateCounters(updCntrs);
+ }
+
/**
* Post-lock closure alias.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java
index b61b1a9..651be60 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java
@@ -56,4 +56,12 @@ public interface IgniteTxLocalEx extends IgniteInternalTx {
* @throws IgniteCheckedException If finish failed.
*/
public boolean localFinish(boolean commit, boolean clearThreadMap) throws IgniteCheckedException;
+
+ /**
+ * Remembers that particular cache partition was touched by current tx.
+ *
+ * @param cacheId Cache id.
+ * @param partId Partition id.
+ */
+ public void touchPartition(int cacheId, int partId);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalState.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalState.java
index 123d396..01eb4f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalState.java
@@ -17,6 +17,9 @@
package org.apache.ignite.internal.processors.cache.transactions;
+import java.util.Map;
+import java.util.Set;
+
/**
*
*/
@@ -41,4 +44,17 @@ public interface IgniteTxLocalState extends IgniteTxState {
*
*/
public void seal();
+
+ /**
+ * @return Cache partitions touched by current tx.
+ */
+ public Map<Integer, Set<Integer>> touchedPartitions();
+
+ /**
+ * Remembers that particular cache partition was touched by current tx.
+ *
+ * @param cacheId Cache id.
+ * @param partId Partition id.
+ */
+ public void touchPartition(int cacheId, int partId);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalStateAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalStateAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalStateAdapter.java
index 4943aac..9c6ef8f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalStateAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalStateAdapter.java
@@ -17,7 +17,13 @@
package org.apache.ignite.internal.processors.cache.transactions;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -25,6 +31,11 @@ import org.apache.ignite.internal.util.typedef.internal.U;
*
*/
public abstract class IgniteTxLocalStateAdapter implements IgniteTxLocalState {
+ /** */
+ private static final Function<Integer, Set<Integer>> CREATE_INT_SET = k -> new HashSet<>();
+ /** */
+ private Map<Integer, Set<Integer>> touchedParts;
+
/**
* @param cacheCtx Cache context.
* @param tx Transaction.
@@ -40,4 +51,19 @@ public abstract class IgniteTxLocalStateAdapter implements IgniteTxLocalState {
cacheCtx.cache().metrics0().onTxRollback(durationNanos);
}
}
+
+ /** {@inheritDoc} */
+ @Override public Map<Integer, Set<Integer>> touchedPartitions() {
+ Map<Integer, Set<Integer>> parts = touchedParts;
+
+ return parts != null ? Collections.unmodifiableMap(parts) : null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void touchPartition(int cacheId, int partId) {
+ if (touchedParts == null)
+ touchedParts = new HashMap<>();
+
+ touchedParts.computeIfAbsent(cacheId, CREATE_INT_SET).add(partId);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteEx.java
index 1e0645f..87cc7cc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteEx.java
@@ -18,9 +18,7 @@
package org.apache.ignite.internal.processors.cache.transactions;
import java.util.Collection;
-import java.util.Map;
import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionsUpdateCountersMap;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
/**
@@ -52,14 +50,4 @@ public interface IgniteTxRemoteEx extends IgniteInternalTx {
* @param cntrs Partition update indexes.
*/
public void setPartitionUpdateCounters(long[] cntrs);
-
- /**
- * @param updCntrsMap Partition update counters map: cacheId -> partId -> updateCntr.
- */
- public void updateCountersMap(Map<Integer, GridDhtPartitionsUpdateCountersMap> updCntrsMap);
-
- /**
- * @return Partition update counters map: cacheId -> partId -> updateCntr.
- */
- public Map<Integer, GridDhtPartitionsUpdateCountersMap> updateCountersMap();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxCounters.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxCounters.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxCounters.java
new file mode 100644
index 0000000..2ad4f94
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxCounters.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.processors.cache.distributed.dht.PartitionUpdateCounters;
+
+/**
+ * Values which should be tracked during transaction execution and applied on commit.
+ */
+public class TxCounters {
+ /** Size changes for cache partitions made by transaction */
+ private final ConcurrentMap<Integer, ConcurrentMap<Integer, AtomicLong>> sizeDeltas = new ConcurrentHashMap<>();
+ /** Update counters for cache partitions in the end of transaction */
+ private Map<Integer, PartitionUpdateCounters> updCntrs;
+
+ /**
+ * Accumulates size change for cache partition.
+ *
+ * @param cacheId Cache id.
+ * @param part Partition id.
+ * @param delta Size delta.
+ */
+ public void accumulateSizeDelta(int cacheId, int part, long delta) {
+ ConcurrentMap<Integer, AtomicLong> partDeltas = sizeDeltas.get(cacheId);
+
+ if (partDeltas == null) {
+ ConcurrentMap<Integer, AtomicLong> partDeltas0 =
+ sizeDeltas.putIfAbsent(cacheId, partDeltas = new ConcurrentHashMap<>());
+
+ if (partDeltas0 != null)
+ partDeltas = partDeltas0;
+ }
+
+ AtomicLong accDelta = partDeltas.get(part);
+
+ if (accDelta == null) {
+ AtomicLong accDelta0 = partDeltas.putIfAbsent(part, accDelta = new AtomicLong());
+
+ if (accDelta0 != null)
+ accDelta = accDelta0;
+ }
+
+ // here AtomicLong is used more as a container,
+ // every instance is assumed to be accessed in thread-confined manner
+ accDelta.set(accDelta.get() + delta);
+ }
+
+ /** */
+ public void updateCounters(Map<Integer, PartitionUpdateCounters> updCntrs) {
+ this.updCntrs = updCntrs;
+ }
+
+ /** */
+ public Map<Integer, PartitionUpdateCounters> updateCounters() {
+ return updCntrs != null ? Collections.unmodifiableMap(updCntrs) : Collections.emptyMap();
+ }
+
+ /** */
+ public Map<Integer, ? extends Map<Integer, AtomicLong>> sizeDeltas() {
+ return Collections.unmodifiableMap(sizeDeltas);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSizeTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSizeTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSizeTest.java
new file mode 100644
index 0000000..32709ff
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSizeTest.java
@@ -0,0 +1,488 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.mvcc;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.function.Consumer;
+import java.util.stream.IntStream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
+
+import static org.apache.ignite.cache.CachePeekMode.BACKUP;
+
+/**
+ *
+ */
+public class CacheMvccSizeTest extends CacheMvccAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected CacheMode cacheMode() {
+ return CacheMode.PARTITIONED;
+ }
+
+ /** */
+ private void checkSizeModificationByOperation(String sql, boolean commit, int expSizeDelta) throws Exception {
+ checkSizeModificationByOperation(c -> {}, cache -> cache.query(q(sql)).getAll(), commit, expSizeDelta);
+ }
+
+ /** */
+ private void checkSizeModificationByOperation(String initSql, String sql, boolean commit,
+ int expSizeDelta) throws Exception {
+ checkSizeModificationByOperation(
+ cache -> cache.query(q(initSql)).getAll(),
+ cache -> cache.query(q(sql)).getAll(),
+ commit,
+ expSizeDelta);
+ }
+
+ /** */
+ private void checkSizeModificationByOperation(Consumer<IgniteCache<?, ?>> inTx, boolean commit,
+ int expSizeDelta) throws Exception {
+ checkSizeModificationByOperation(c -> {}, inTx, commit, expSizeDelta);
+ }
+
+ /** */
+ private void checkSizeModificationByOperation(Consumer<IgniteCache<?, ?>> beforeTx,
+ Consumer<IgniteCache<?, ?>> inTx, boolean commit, int expSizeDelta) throws Exception {
+ IgniteCache<Object, Object> tbl0 = grid(0).cache("person");
+
+ tbl0.query(q("delete from person"));
+
+ beforeTx.accept(tbl0);
+
+ int initSize = tbl0.size();
+
+ tbl0.query(q("begin"));
+
+ inTx.accept(tbl0);
+
+ // size is not changed before commit
+ assertEquals(0, tbl0.size() - initSize);
+
+ if (commit)
+ tbl0.query(q("commit"));
+ else
+ tbl0.query(q("rollback"));
+
+ assertEquals(expSizeDelta, tbl0.size() - initSize);
+ assertEquals(tbl0.size(), table(grid(1)).size());
+
+ assertEquals(tbl0.size(), tbl0.size(BACKUP));
+ assertEquals(tbl0.size(), table(grid(1)).size(BACKUP));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testSql() throws Exception {
+ startGridsMultiThreaded(2);
+
+ createTable(grid(0));
+
+ checkSizeModificationByOperation("insert into person values(1, 'a')", true, 1);
+
+ checkSizeModificationByOperation("insert into person values(1, 'a')", false, 0);
+
+ checkSizeModificationByOperation(
+ personTbl -> personTbl.query(q("insert into person values(1, 'a')")),
+ personTbl -> {
+ try {
+ personTbl.query(q("insert into person values(1, 'a')"));
+ }
+ catch (Exception e) {
+ if (e.getCause() instanceof IgniteSQLException) {
+ assertEquals(IgniteQueryErrorCode.DUPLICATE_KEY,
+ ((IgniteSQLException)e.getCause()).statusCode());
+ }
+ else {
+ e.printStackTrace();
+
+ fail("Unexpected exceptions");
+ }
+ }
+ },
+ true, 0);
+
+ checkSizeModificationByOperation("merge into person(id, name) values(1, 'a')", true, 1);
+
+ checkSizeModificationByOperation("merge into person(id, name) values(1, 'a')", false, 0);
+
+ checkSizeModificationByOperation(
+ "insert into person values(1, 'a')", "merge into person(id, name) values(1, 'b')", true, 0);
+
+ checkSizeModificationByOperation("update person set name = 'b' where id = 1", true, 0);
+
+ checkSizeModificationByOperation(
+ "insert into person values(1, 'a')", "update person set name = 'b' where id = 1", true, 0);
+
+ checkSizeModificationByOperation(
+ "insert into person values(1, 'a')", "delete from person where id = 1", true, -1);
+
+ checkSizeModificationByOperation(
+ "insert into person values(1, 'a')", "delete from person where id = 1", false, 0);
+
+ checkSizeModificationByOperation("delete from person where id = 1", true, 0);
+
+ checkSizeModificationByOperation(
+ "insert into person values(1, 'a')", "select * from person", true, 0);
+
+ checkSizeModificationByOperation("select * from person", true, 0);
+
+ checkSizeModificationByOperation(
+ "insert into person values(1, 'a')", "select * from person where id = 1 for update", true, 0);
+
+ checkSizeModificationByOperation("select * from person where id = 1 for update", true, 0);
+
+ checkSizeModificationByOperation(personTbl -> {
+ personTbl.query(q("insert into person values(1, 'a')"));
+
+ personTbl.query(q("insert into person values(%d, 'b')", keyInSamePartition(grid(0), "person", 1)));
+
+ personTbl.query(q("insert into person values(%d, 'c')", keyInDifferentPartition(grid(0), "person", 1)));
+ }, true, 3);
+
+ checkSizeModificationByOperation(personTbl -> {
+ personTbl.query(q("insert into person values(1, 'a')"));
+
+ personTbl.query(q("delete from person where id = 1"));
+ }, true, 0);
+
+ checkSizeModificationByOperation(personTbl -> {
+ personTbl.query(q("insert into person values(1, 'a')"));
+
+ personTbl.query(q("delete from person where id = 1"));
+
+ personTbl.query(q("insert into person values(1, 'a')"));
+ }, true, 1);
+
+ checkSizeModificationByOperation(
+ personTbl -> personTbl.query(q("insert into person values(1, 'a')")),
+ personTbl -> {
+ personTbl.query(q("delete from person where id = 1"));
+
+ personTbl.query(q("insert into person values(1, 'a')"));
+ }, true, 0);
+
+ checkSizeModificationByOperation(personTbl -> {
+ personTbl.query(q("merge into person(id, name) values(1, 'a')"));
+
+ personTbl.query(q("delete from person where id = 1"));
+ }, true, 0);
+
+ checkSizeModificationByOperation(personTbl -> {
+ personTbl.query(q("merge into person(id, name) values(1, 'a')"));
+
+ personTbl.query(q("delete from person where id = 1"));
+
+ personTbl.query(q("merge into person(id, name) values(1, 'a')"));
+ }, true, 1);
+
+ checkSizeModificationByOperation(
+ personTbl -> personTbl.query(q("merge into person(id, name) values(1, 'a')")),
+ personTbl -> {
+ personTbl.query(q("delete from person where id = 1"));
+
+ personTbl.query(q("merge into person(id, name) values(1, 'a')"));
+ }, true, 0);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testInsertDeleteConcurrent() throws Exception {
+ startGridsMultiThreaded(2);
+
+ IgniteCache<?, ?> tbl0 = createTable(grid(0));
+
+ SqlFieldsQuery insert = new SqlFieldsQuery("insert into person(id, name) values(?, 'a')");
+
+ SqlFieldsQuery delete = new SqlFieldsQuery("delete from person where id = ?");
+
+ CompletableFuture<Integer> insertFut = CompletableFuture.supplyAsync(() -> {
+ int cnt = 0;
+
+ for (int i = 0; i < 1000; i++)
+ cnt += update(insert.setArgs(ThreadLocalRandom.current().nextInt(10)), tbl0);
+
+ return cnt;
+ });
+
+ CompletableFuture<Integer> deleteFut = CompletableFuture.supplyAsync(() -> {
+ int cnt = 0;
+
+ for (int i = 0; i < 1000; i++)
+ cnt += update(delete.setArgs(ThreadLocalRandom.current().nextInt(10)), tbl0);
+
+ return cnt;
+ });
+
+ int expSize = insertFut.join() - deleteFut.join();
+
+ assertEquals(expSize, tbl0.size());
+ assertEquals(expSize, table(grid(1)).size());
+
+ assertEquals(expSize, tbl0.size(BACKUP));
+ assertEquals(expSize, table(grid(1)).size(BACKUP));
+ }
+
+ /** */
+ private int update(SqlFieldsQuery qry, IgniteCache<?, ?> cache) {
+ try {
+ return Integer.parseInt(cache.query(qry).getAll().get(0).get(0).toString());
+ } catch (Exception e) {
+ return 0;
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWriteConflictDoesNotChangeSize() throws Exception {
+ startGridsMultiThreaded(2);
+
+ IgniteCache<?, ?> tbl0 = createTable(grid(0));
+
+ tbl0.query(q("insert into person values(1, 'a')"));
+
+ tbl0.query(q("begin"));
+
+ tbl0.query(q("delete from person where id = 1"));
+
+ CompletableFuture<Void> conflictingStarted = new CompletableFuture<>();
+
+ CompletableFuture<Void> fut = CompletableFuture.runAsync(() -> {
+ tbl0.query(q("begin"));
+
+ try {
+ tbl0.query(q("select * from person")).getAll();
+ conflictingStarted.complete(null);
+
+ tbl0.query(q("merge into person(id, name) values(1, 'b')"));
+ }
+ finally {
+ tbl0.query(q("commit"));
+ }
+ });
+
+ conflictingStarted.join();
+ tbl0.query(q("commit"));
+
+ try {
+ fut.join();
+ }
+ catch (Exception e) {
+ if (e.getCause().getCause() instanceof IgniteSQLException)
+ assertTrue(e.getMessage().toLowerCase().contains("version mismatch"));
+ else {
+ e.printStackTrace();
+
+ fail("Unexpected exception");
+ }
+ }
+
+ assertEquals(0, tbl0.size());
+ assertEquals(0, table(grid(1)).size());
+
+ assertEquals(0, tbl0.size(BACKUP));
+ assertEquals(0, table(grid(1)).size(BACKUP));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testDeleteChangesSizeAfterUnlock() throws Exception {
+ startGridsMultiThreaded(2);
+
+ IgniteCache<?, ?> tbl0 = createTable(grid(0));
+
+ tbl0.query(q("insert into person values(1, 'a')"));
+
+ tbl0.query(q("begin"));
+
+ tbl0.query(q("select * from person where id = 1 for update")).getAll();
+
+ CompletableFuture<Thread> asyncThread = new CompletableFuture<>();
+
+ CompletableFuture<Void> fut = CompletableFuture.runAsync(() -> {
+ tbl0.query(q("begin"));
+
+ try {
+ tbl0.query(q("select * from person")).getAll();
+
+ asyncThread.complete(Thread.currentThread());
+ tbl0.query(q("delete from person where id = 1"));
+ }
+ finally {
+ tbl0.query(q("commit"));
+ }
+ });
+
+ Thread concThread = asyncThread.join();
+
+ // wait until concurrent thread blocks awaiting entry mvcc lock release
+ while (concThread.getState() == Thread.State.RUNNABLE && !Thread.currentThread().isInterrupted());
+
+ tbl0.query(q("commit"));
+
+ fut.join();
+
+ assertEquals(0, tbl0.size());
+ assertEquals(0, table(grid(1)).size());
+
+ assertEquals(0, tbl0.size(BACKUP));
+ assertEquals(0, table(grid(1)).size(BACKUP));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testDataStreamerModifiesReplicatedCacheSize() throws Exception {
+ startGridsMultiThreaded(2);
+
+ IgniteEx ignite = grid(0);
+
+ ignite.createCache(
+ new CacheConfiguration<>("test")
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setCacheMode(CacheMode.REPLICATED)
+ );
+
+ try (IgniteDataStreamer<Object, Object> streamer = ignite.dataStreamer("test")) {
+ streamer.addData(1, "a");
+
+ streamer.addData(keyInDifferentPartition(ignite, "test", 1), "b");
+ }
+
+ assertEquals(2, ignite.cache("test").size());
+
+ assertEquals(1, grid(0).cache("test").localSize());
+ assertEquals(1, grid(0).cache("test").localSize(BACKUP));
+
+ assertEquals(1, grid(1).cache("test").localSize());
+ assertEquals(1, grid(1).cache("test").localSize(BACKUP));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testSizeIsConsistentAfterRebalance() throws Exception {
+ IgniteEx ignite = startGrid(0);
+
+ IgniteCache<?, ?> tbl = createTable(ignite);
+
+ for (int i = 0; i < 100; i++)
+ tbl.query(q("insert into person values(?, ?)").setArgs(i, i));
+
+ startGrid(1);
+
+ awaitPartitionMapExchange();
+
+ IgniteCache<?, ?> tbl0 = grid(0).cache("person");
+ IgniteCache<?, ?> tbl1 = grid(1).cache("person");
+
+ assert tbl0.localSize() != 0 && tbl1.localSize() != 0;
+
+ assertEquals(100, tbl1.size());
+ assertEquals(100, tbl0.localSize() + tbl1.localSize());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSizeIsConsistentAfterRebalanceDuringInsert() throws Exception {
+ IgniteEx ignite = startGrid(0);
+
+ IgniteCache<?, ?> tbl = createTable(ignite);
+
+ Future<?> f = null;
+
+ for (int i = 0; i < 100; i++) {
+ if (i == 50)
+ f = ForkJoinPool.commonPool().submit(() -> startGrid(1));
+
+ tbl.query(q("insert into person values(?, ?)").setArgs(i, i));
+ }
+
+ f.get();
+
+ awaitPartitionMapExchange();
+
+ IgniteCache<?, ?> tbl0 = grid(0).cache("person");
+ IgniteCache<?, ?> tbl1 = grid(1).cache("person");
+
+ assert tbl0.localSize() != 0 && tbl1.localSize() != 0;
+
+ assertEquals(100, tbl1.size());
+ assertEquals(100, tbl0.localSize() + tbl1.localSize());
+ }
+
+ /** */
+ private static IgniteCache<?, ?> table(IgniteEx ignite) {
+ assert ignite.cachex("person").configuration().getAtomicityMode() == CacheAtomicityMode.TRANSACTIONAL;
+ assert ignite.cachex("person").configuration().getCacheMode() == CacheMode.REPLICATED;
+
+ return ignite.cache("person");
+ }
+
+ /** */
+ private static IgniteCache<?, ?> createTable(IgniteEx ignite) {
+ IgniteCache<?, ?> sqlNexus = ignite.getOrCreateCache(new CacheConfiguration<>("sqlNexus").setSqlSchema("PUBLIC"));
+
+ sqlNexus.query(q("" +
+ "create table person(" +
+ " id int primary key," +
+ " name varchar" +
+ ") with \"atomicity=transactional,template=replicated,cache_name=person\""));
+
+ return table(ignite);
+ }
+
+ /** */
+ private static SqlFieldsQuery q(String fSql, Object... args) {
+ return new SqlFieldsQuery(String.format(fSql, args));
+ }
+
+ /** */
+ private static int keyInSamePartition(Ignite ignite, String cacheName, int key) {
+ Affinity<Object> affinity = ignite.affinity(cacheName);
+
+ return IntStream.iterate(key + 1, i -> i + 1)
+ .filter(i -> affinity.partition(i) == affinity.partition(key))
+ .findFirst().getAsInt();
+ }
+
+ /** */
+ private static int keyInDifferentPartition(Ignite ignite, String cacheName, int key) {
+ Affinity<Object> affinity = ignite.affinity(cacheName);
+
+ return IntStream.iterate(key + 1, i -> i + 1)
+ .filter(i -> affinity.partition(i) != affinity.partition(key))
+ .findFirst().getAsInt();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00e935d3/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java
index ab9d2e6..c8f7643 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java
@@ -36,6 +36,7 @@ import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccReplicatedSqlQu
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccReplicatedSqlTxQueriesTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccReplicatedSqlTxQueriesWithReducerTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccScanQueryWithConcurrentJdbcTransactionTest;
+import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccSizeTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccSizeWithConcurrentJdbcTransactionTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccStreamingInsertTest;
import org.apache.ignite.internal.processors.query.h2.GridIndexRebuildWithMvccEnabledSelfTest;
@@ -50,9 +51,10 @@ public class IgniteCacheMvccSqlTestSuite extends TestSuite {
public static TestSuite suite() {
TestSuite suite = new TestSuite("IgniteCache SQL MVCC Test Suite");
- // Simle tests.
+ // Simple tests.
suite.addTestSuite(CacheMvccDmlSimpleTest.class);
suite.addTestSuite(SqlTransactionsCommandsWithMvccEnabledSelfTest.class);
+ suite.addTestSuite(CacheMvccSizeTest.class);
suite.addTestSuite(GridIndexRebuildWithMvccEnabledSelfTest.class);
@@ -83,7 +85,6 @@ public class IgniteCacheMvccSqlTestSuite extends TestSuite {
suite.addTestSuite(CacheMvccPartitionedSqlCoordinatorFailoverTest.class);
suite.addTestSuite(CacheMvccReplicatedSqlCoordinatorFailoverTest.class);
-
return suite;
}
}
[02/50] [abbrv] ignite git commit: IGNITE-9444 Mute failing test
Posted by ag...@apache.org.
IGNITE-9444 Mute failing test
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/271c7cd0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/271c7cd0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/271c7cd0
Branch: refs/heads/ignite-5960
Commit: 271c7cd00ae3aab4e28f07da217585a3e33f0620
Parents: dcefc16
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Fri Aug 31 13:28:40 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Aug 31 13:28:40 2018 +0300
----------------------------------------------------------------------
.../tcp/ipfinder/cloud/TcpDiscoveryCloudIpFinderSelfTest.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/271c7cd0/modules/cloud/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudIpFinderSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/cloud/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudIpFinderSelfTest.java b/modules/cloud/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudIpFinderSelfTest.java
index 4bddb18..c754553 100644
--- a/modules/cloud/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudIpFinderSelfTest.java
+++ b/modules/cloud/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudIpFinderSelfTest.java
@@ -81,6 +81,8 @@ public class TcpDiscoveryCloudIpFinderSelfTest extends
* @throws Exception If any error occurs.
*/
public void testRackspace() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-9444");
+
testCloudProvider("rackspace-cloudservers-us");
}
[22/50] [abbrv] ignite git commit: IGNITE-8987 Fixed Ignite hang on
getting of atomic structures after autoactivation - Fixes #4664.
Posted by ag...@apache.org.
IGNITE-8987 Fixed Ignite hang on getting of atomic structures after autoactivation - Fixes #4664.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/02afb437
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/02afb437
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/02afb437
Branch: refs/heads/ignite-5960
Commit: 02afb437f4286c421e53de732c3041e182817c5d
Parents: dfaffe2
Author: Roman Guseinov <gr...@gmail.com>
Authored: Tue Sep 4 16:20:16 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Sep 4 16:20:16 2018 +0300
----------------------------------------------------------------------
.../datastructures/DataStructuresProcessor.java | 5 ++-
...IgnitePersistentStoreDataStructuresTest.java | 45 +++++++++++++++++++-
2 files changed, 48 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/02afb437/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
index 8f6876c..2c77f17 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
@@ -193,7 +193,10 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
*
*/
public void onBeforeActivate() {
- initLatch = new CountDownLatch(1);
+ CountDownLatch latch0 = initLatch;
+
+ if (latch0 == null || latch0.getCount() == 0)
+ initLatch = new CountDownLatch(1);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/02afb437/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
index 53e014e..855b035 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache.persistence;
+import java.util.concurrent.TimeUnit;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteAtomicLong;
import org.apache.ignite.IgniteAtomicSequence;
@@ -30,6 +31,8 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -43,6 +46,9 @@ public class IgnitePersistentStoreDataStructuresTest extends GridCommonAbstractT
/** */
private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+ /** */
+ private static volatile boolean autoActivationEnabled = false;
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -56,7 +62,7 @@ public class IgnitePersistentStoreDataStructuresTest extends GridCommonAbstractT
cfg.setDataStorageConfiguration(memCfg);
- cfg.setAutoActivationEnabled(false);
+ cfg.setAutoActivationEnabled(autoActivationEnabled);
return cfg;
}
@@ -71,6 +77,8 @@ public class IgnitePersistentStoreDataStructuresTest extends GridCommonAbstractT
super.beforeTest();
cleanPersistenceDir();
+
+ autoActivationEnabled = false;
}
/** {@inheritDoc} */
@@ -162,6 +170,41 @@ public class IgnitePersistentStoreDataStructuresTest extends GridCommonAbstractT
/**
* @throws Exception If failed.
*/
+ public void testSequenceAfterAutoactivation() throws Exception {
+ final String seqName = "testSequence";
+
+ autoActivationEnabled = true;
+
+ Ignite ignite = startGrids(2);
+
+ ignite.cluster().active(true);
+
+ ignite.atomicSequence(seqName, 0, true);
+
+ stopAllGrids(true);
+
+ final Ignite node = startGrids(2);
+
+ IgniteInternalFuture fut = GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ // Should not hang.
+ node.atomicSequence(seqName, 0, false);
+ }
+ });
+
+ try {
+ fut.get(10, TimeUnit.SECONDS);
+ }
+ catch (IgniteFutureTimeoutCheckedException e) {
+ fut.cancel();
+
+ fail("Ignite was stuck on getting the atomic sequence after autoactivation.");
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testSet() throws Exception {
fail("https://issues.apache.org/jira/browse/IGNITE-5553");
[17/50] [abbrv] ignite git commit: IGNITE-9141: SQL: pass error
message from mapper to reducer in case of mapping failure. This closes #4536.
Posted by ag...@apache.org.
IGNITE-9141: SQL: pass error message from mapper to reducer in case of mapping failure. This closes #4536.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/525a7754
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/525a7754
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/525a7754
Branch: refs/heads/ignite-5960
Commit: 525a77547d7f98b1a88ba98f60f328da3aa4947a
Parents: 08c563a
Author: SGrimstad <sg...@gridgain.com>
Authored: Tue Sep 4 11:57:07 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Tue Sep 4 11:57:07 2018 +0300
----------------------------------------------------------------------
.../messages/GridQueryNextPageResponse.java | 35 +-
.../query/h2/twostep/GridMapQueryExecutor.java | 131 +++---
.../h2/twostep/GridReduceQueryExecutor.java | 75 ++--
.../query/h2/twostep/GridResultPage.java | 2 -
.../query/h2/twostep/ReduceQueryRun.java | 117 +++++-
...sappearedCacheCauseRetryMessageSelfTest.java | 134 ++++++
...appearedCacheWasNotFoundMessageSelfTest.java | 123 ++++++
.../query/h2/twostep/JoinSqlTestHelper.java | 163 ++++++++
.../NonCollocatedRetryMessageSelfTest.java | 146 +++++++
.../h2/twostep/RetryCauseMessageSelfTest.java | 416 +++++++++++++++++++
.../IgniteCacheQuerySelfTestSuite2.java | 9 +
11 files changed, 1236 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/525a7754/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
index 2d1ec36..6b976c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
@@ -67,6 +67,9 @@ public class GridQueryNextPageResponse implements Message {
/** */
private AffinityTopologyVersion retry;
+ /** Retry cause description*/
+ private String retryCause;
+
/** Last page flag. */
private boolean last;
@@ -235,6 +238,12 @@ public class GridQueryNextPageResponse implements Message {
writer.incrementState();
case 9:
+ if (!writer.writeString("retryCause", retryCause))
+ return false;
+
+ writer.incrementState();
+
+ case 10:
if (!writer.writeBoolean("removeMapping", removeMapping))
return false;
@@ -323,13 +332,23 @@ public class GridQueryNextPageResponse implements Message {
return false;
reader.incrementState();
+
case 9:
+ retryCause = reader.readString("retryCause");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 10:
removeMapping = reader.readBoolean("removeMapping");
if (!reader.isLastRead())
return false;
reader.incrementState();
+
}
return reader.afterMessageRead(GridQueryNextPageResponse.class);
@@ -342,7 +361,7 @@ public class GridQueryNextPageResponse implements Message {
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 10;
+ return 11;
}
/**
@@ -360,6 +379,20 @@ public class GridQueryNextPageResponse implements Message {
}
/**
+ * @return Retry Ccause message.
+ */
+ public String retryCause() {
+ return retryCause;
+ }
+
+ /**
+ * @param retryCause Retry Ccause message.
+ */
+ public void retryCause(String retryCause){
+ this.retryCause = retryCause;
+ }
+
+ /**
* @return Last page flag.
*/
public boolean last() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/525a7754/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index 2402247..b4d6f00 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -312,10 +312,10 @@ public class GridMapQueryExecutor {
* @param reserved Reserved list.
* @param nodeId Node ID.
* @param reqId Request ID.
- * @return {@code true} If all the needed partitions successfully reserved.
+ * @return String which is null in case of success or with causeMessage if failed
* @throws IgniteCheckedException If failed.
*/
- private boolean reservePartitions(
+ private String reservePartitions(
@Nullable List<Integer> cacheIds,
AffinityTopologyVersion topVer,
final int[] explicitParts,
@@ -326,7 +326,7 @@ public class GridMapQueryExecutor {
assert topVer != null;
if (F.isEmpty(cacheIds))
- return true;
+ return null;
Collection<Integer> partIds = wrap(explicitParts);
@@ -335,11 +335,11 @@ public class GridMapQueryExecutor {
// Cache was not found, probably was not deployed yet.
if (cctx == null) {
- logRetry("Failed to reserve partitions for query (cache is not found on local node) [" +
- "rmtNodeId=" + nodeId + ", reqId=" + reqId + ", affTopVer=" + topVer + ", cacheId=" +
- cacheIds.get(i) + "]");
+ final String res = String.format("Failed to reserve partitions for query (cache is not found on " +
+ "local node) [localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s]",
+ ctx.localNodeId(), nodeId, reqId, topVer, cacheIds.get(i));
- return false;
+ return res;
}
if (cctx.isLocal() || !cctx.rebalanceEnabled())
@@ -352,13 +352,10 @@ public class GridMapQueryExecutor {
if (explicitParts == null && r != null) { // Try to reserve group partition if any and no explicits.
if (r != MapReplicatedReservation.INSTANCE) {
- if (!r.reserve()) {
- logRetry("Failed to reserve partitions for query (group reservation failed) [" +
- "rmtNodeId=" + nodeId + ", reqId=" + reqId + ", affTopVer=" + topVer +
- ", cacheId=" + cacheIds.get(i) + ", cacheName=" + cctx.name() + "]");
-
- return false; // We need explicit partitions here -> retry.
- }
+ if (!r.reserve())
+ return String.format("Failed to reserve partitions for query (group " +
+ "reservation failed) [localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, " +
+ "cacheName=%s]",ctx.localNodeId(), nodeId, reqId, topVer, cacheIds.get(i), cctx.name());
reserved.add(r);
}
@@ -374,15 +371,21 @@ public class GridMapQueryExecutor {
// We don't need to reserve partitions because they will not be evicted in replicated caches.
GridDhtPartitionState partState = part != null ? part.state() : null;
- if (partState != OWNING) {
- logRetry("Failed to reserve partitions for query (partition of " +
- "REPLICATED cache is not in OWNING state) [rmtNodeId=" + nodeId +
- ", reqId=" + reqId + ", affTopVer=" + topVer + ", cacheId=" + cacheIds.get(i) +
- ", cacheName=" + cctx.name() + ", part=" + p + ", partFound=" + (part != null) +
- ", partState=" + partState + "]");
-
- return false;
- }
+ if (partState != OWNING)
+ return String.format("Failed to reserve partitions for query " +
+ "(partition of REPLICATED cache is not in OWNING state) [" +
+ "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, cacheName=%s, " +
+ "part=%s, partFound=%s, partState=%s]",
+ ctx.localNodeId(),
+ nodeId,
+ reqId,
+ topVer,
+ cacheIds.get(i),
+ cctx.name(),
+ p,
+ (part != null),
+ partState
+ );
}
// Mark that we checked this replicated cache.
@@ -398,29 +401,41 @@ public class GridMapQueryExecutor {
GridDhtPartitionState partState = part != null ? part.state() : null;
- if (partState != OWNING || !part.reserve()) {
- logRetry("Failed to reserve partitions for query (partition of " +
- "PARTITIONED cache cannot be reserved) [rmtNodeId=" + nodeId + ", reqId=" + reqId +
- ", affTopVer=" + topVer + ", cacheId=" + cacheIds.get(i) +
- ", cacheName=" + cctx.name() + ", part=" + partId + ", partFound=" + (part != null) +
- ", partState=" + partState + "]");
-
- return false;
- }
+ if (partState != OWNING || !part.reserve())
+ return String.format("Failed to reserve partitions for query " +
+ "(partition of PARTITIONED cache cannot be reserved) [" +
+ "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, cacheName=%s, " +
+ "part=%s, partFound=%s, partState=%s]",
+ ctx.localNodeId(),
+ nodeId,
+ reqId,
+ topVer,
+ cacheIds.get(i),
+ cctx.name(),
+ partId,
+ (part != null),
+ partState
+ );
reserved.add(part);
// Double check that we are still in owning state and partition contents are not cleared.
partState = part.state();
- if (part.state() != OWNING) {
- logRetry("Failed to reserve partitions for query (partition of " +
- "PARTITIONED cache is not in OWNING state after reservation) [rmtNodeId=" + nodeId +
- ", reqId=" + reqId + ", affTopVer=" + topVer + ", cacheId=" + cacheIds.get(i) +
- ", cacheName=" + cctx.name() + ", part=" + partId + ", partState=" + partState + "]");
-
- return false;
- }
+ if (part.state() != OWNING)
+ return String.format("Failed to reserve partitions for query " +
+ "(partition of PARTITIONED cache is not in OWNING state after reservation) [" +
+ "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, cacheName=%s, " +
+ "part=%s, partState=%s]",
+ ctx.localNodeId(),
+ nodeId,
+ reqId,
+ topVer,
+ cacheIds.get(i),
+ cctx.name(),
+ partId,
+ partState
+ );
}
if (explicitParts == null) {
@@ -442,16 +457,7 @@ public class GridMapQueryExecutor {
}
}
- return true;
- }
-
- /**
- * Load failed partition reservation.
- *
- * @param msg Message.
- */
- private void logRetry(String msg) {
- log.info(msg);
+ return null;
}
/**
@@ -783,12 +789,14 @@ public class GridMapQueryExecutor {
// otherwise, their state is protected by locked topology.
if (topVer != null && txDetails == null) {
// Reserve primary for topology version or explicit partitions.
- if (!reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId)) {
+ String err = reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId);
+
+ if (!F.isEmpty(err)) {
// Unregister lazy worker because re-try may never reach this node again.
if (lazy)
stopAndUnregisterCurrentLazyWorker();
- sendRetry(node, reqId, segmentId);
+ sendRetry(node, reqId, segmentId, err);
return;
}
@@ -975,10 +983,12 @@ public class GridMapQueryExecutor {
GridH2RetryException retryErr = X.cause(e, GridH2RetryException.class);
if (retryErr != null) {
- logRetry("Failed to execute non-collocated query (will retry) [nodeId=" + node.id() +
- ", reqId=" + reqId + ", errMsg=" + retryErr.getMessage() + ']');
+ final String retryCause = String.format(
+ "Failed to execute non-collocated query (will retry) [localNodeId=%s, rmtNodeId=%s, reqId=%s, " +
+ "errMsg=%s]", ctx.localNodeId(), node.id(), reqId, retryErr.getMessage()
+ );
- sendRetry(node, reqId, segmentId);
+ sendRetry(node, reqId, segmentId, retryCause);
}
else {
U.error(log, "Failed to execute local query.", e);
@@ -1035,13 +1045,15 @@ public class GridMapQueryExecutor {
List<GridReservable> reserved = new ArrayList<>();
- if (!reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId)) {
+ String err = reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId);
+
+ if (!F.isEmpty(err)) {
U.error(log, "Failed to reserve partitions for DML request. [localNodeId=" + ctx.localNodeId() +
", nodeId=" + node.id() + ", reqId=" + req.requestId() + ", cacheIds=" + cacheIds +
", topVer=" + topVer + ", parts=" + Arrays.toString(parts) + ']');
- sendUpdateResponse(node, reqId, null, "Failed to reserve partitions for DML request. " +
- "Explanation (Retry your request when re-balancing is over).");
+ sendUpdateResponse(node, reqId, null,
+ "Failed to reserve partitions for DML request. " + err);
return;
}
@@ -1299,7 +1311,7 @@ public class GridMapQueryExecutor {
* @param reqId Request ID.
* @param segmentId Index segment ID.
*/
- private void sendRetry(ClusterNode node, long reqId, int segmentId) {
+ private void sendRetry(ClusterNode node, long reqId, int segmentId, String retryCause) {
try {
boolean loc = node.isLocal();
@@ -1310,6 +1322,7 @@ public class GridMapQueryExecutor {
false);
msg.retry(h2.readyTopologyVersion());
+ msg.retryCause(retryCause);
if (loc)
h2.reduceQueryExecutor().onMessage(ctx.localNodeId(), msg);
http://git-wip-us.apache.org/repos/asf/ignite/blob/525a7754/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 910ad1a..586fb51 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -65,7 +65,6 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLo
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxSelectForUpdateFuture;
import org.apache.ignite.internal.processors.cache.distributed.near.TxTopologyVersionFuture;
import org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker;
-import org.apache.ignite.internal.processors.cache.mvcc.MvccUtils;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
@@ -234,8 +233,7 @@ public class GridReduceQueryExecutor {
* @param nodeId Left node ID.
*/
private void handleNodeLeft(ReduceQueryRun r, UUID nodeId) {
- // Will attempt to retry. If reduce query was started it will fail on next page fetching.
- retry(r, h2.readyTopologyVersion(), nodeId);
+ r.setStateOnNodeLeave(nodeId, h2.readyTopologyVersion());
}
/**
@@ -287,12 +285,13 @@ public class GridReduceQueryExecutor {
*/
private void fail(ReduceQueryRun r, UUID nodeId, String msg, byte failCode) {
if (r != null) {
- CacheException e = new CacheException("Failed to execute map query on the node: " + nodeId + ", " + msg);
+ CacheException e = new CacheException("Failed to execute map query on remote node [nodeId=" + nodeId +
+ ", errMsg=" + msg + ']');
if (failCode == GridQueryFailResponse.CANCELLED_BY_ORIGINATOR)
e.addSuppressed(new QueryCancelledException());
- r.state(e, nodeId);
+ r.setStateOnException(nodeId, e);
}
}
@@ -300,7 +299,7 @@ public class GridReduceQueryExecutor {
* @param node Node.
* @param msg Message.
*/
- private void onNextPage(final ClusterNode node, GridQueryNextPageResponse msg) {
+ private void onNextPage(final ClusterNode node, final GridQueryNextPageResponse msg) {
final long qryReqId = msg.queryRequestId();
final int qry = msg.query();
final int seg = msg.segmentId();
@@ -319,20 +318,13 @@ public class GridReduceQueryExecutor {
try {
page = new GridResultPage(ctx, node.id(), msg) {
@Override public void fetchNextPage() {
- Object errState = r.state();
+ if (r.hasErrorOrRetry()) {
+ if (r.exception() != null)
+ throw r.exception();
- if (errState != null) {
- CacheException err0 = errState instanceof CacheException ? (CacheException)errState : null;
+ assert r.retryCause() != null;
- if (err0 != null && err0.getCause() instanceof IgniteClientDisconnectedException)
- throw err0;
-
- CacheException e = new CacheException("Failed to fetch data from node: " + node.id());
-
- if (err0 != null)
- e.addSuppressed(err0);
-
- throw e;
+ throw new CacheException(r.retryCause());
}
try {
@@ -360,7 +352,7 @@ public class GridReduceQueryExecutor {
idx.addPage(page);
if (msg.retry() != null)
- retry(r, msg.retry(), node.id());
+ r.setStateOnRetry(node.id(), msg.retry(), msg.retryCause());
else if (msg.page() == 0) {
// Do count down on each first page received.
r.latch().countDown();
@@ -373,20 +365,14 @@ public class GridReduceQueryExecutor {
}
/**
- * @param r Query run.
- * @param retryVer Retry version.
- * @param nodeId Node ID.
- */
- private void retry(ReduceQueryRun r, AffinityTopologyVersion retryVer, UUID nodeId) {
- r.state(retryVer, nodeId);
- }
-
- /**
* @param cacheIds Cache IDs.
* @return {@code true} If preloading is active.
*/
private boolean isPreloadingActive(List<Integer> cacheIds) {
for (Integer cacheId : cacheIds) {
+ if (null == cacheContext(cacheId))
+ throw new CacheException(String.format("Cache not found on local node [cacheId=%d]", cacheId));
+
if (hasMovingPartitions(cacheContext(cacheId)))
return true;
}
@@ -399,6 +385,8 @@ public class GridReduceQueryExecutor {
* @return {@code True} If cache has partitions in {@link GridDhtPartitionState#MOVING} state.
*/
private boolean hasMovingPartitions(GridCacheContext<?, ?> cctx) {
+ assert cctx != null;
+
return !cctx.isLocal() && cctx.topology().hasMovingPartitions();
}
@@ -593,9 +581,18 @@ public class GridReduceQueryExecutor {
final long startTime = U.currentTimeMillis();
+ ReduceQueryRun lastRun = null;
+
for (int attempt = 0;; attempt++) {
- if (attempt > 0 && retryTimeout > 0 && (U.currentTimeMillis() - startTime > retryTimeout))
- throw new CacheException("Failed to map SQL query to topology.");
+ if (attempt > 0 && retryTimeout > 0 && (U.currentTimeMillis() - startTime > retryTimeout)) {
+ UUID retryNodeId = lastRun.retryNodeId();
+ String retryCause = lastRun.retryCause();
+
+ assert !F.isEmpty(retryCause);
+
+ throw new CacheException("Failed to map SQL query to topology on data node [dataNodeId=" + retryNodeId +
+ ", msg=" + retryCause + ']');
+ }
if (attempt != 0) {
try {
@@ -877,29 +874,26 @@ public class GridReduceQueryExecutor {
if (send(nodes, req, spec, false)) {
awaitAllReplies(r, nodes, cancel);
- Object state = r.state();
-
- if (state != null) {
- if (state instanceof CacheException) {
- CacheException err = (CacheException)state;
+ if (r.hasErrorOrRetry()) {
+ CacheException err = r.exception();
+ if (err != null) {
if (err.getCause() instanceof IgniteClientDisconnectedException)
throw err;
if (wasCancelled(err))
throw new QueryCancelledException(); // Throw correct exception.
- throw new CacheException("Failed to run map query remotely." + err.getMessage(), err);
+ throw new CacheException("Failed to run map query remotely: " + err.getMessage(), err);
}
-
- if (state instanceof AffinityTopologyVersion) {
+ else {
retry = true;
// On-the-fly topology change must not be possible in FOR UPDATE case.
assert sfuFut == null;
// If remote node asks us to retry then we have outdated full partition map.
- h2.awaitForReadyTopologyVersion((AffinityTopologyVersion)state);
+ h2.awaitForReadyTopologyVersion(r.retryTopologyVersion());
}
}
}
@@ -952,6 +946,9 @@ public class GridReduceQueryExecutor {
}
}
else {
+ assert r != null;
+ lastRun=r;
+
if (Thread.currentThread().isInterrupted())
throw new IgniteInterruptedCheckedException("Query was interrupted.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/525a7754/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridResultPage.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridResultPage.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridResultPage.java
index 3c17640..0cb986b 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridResultPage.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridResultPage.java
@@ -72,8 +72,6 @@ public class GridResultPage {
Collection<?> plainRows = res.plainRows();
if (plainRows != null) {
- assert plainRows instanceof ArrayList;
-
rowsInPage = plainRows.size();
if (rowsInPage == 0 || ((ArrayList<Value[]>)plainRows).get(0).length == res.columns())
http://git-wip-us.apache.org/repos/asf/ignite/blob/525a7754/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReduceQueryRun.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReduceQueryRun.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReduceQueryRun.java
index df72e8c..7ddd653 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReduceQueryRun.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReduceQueryRun.java
@@ -29,6 +29,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxSe
import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
import org.apache.ignite.internal.processors.query.GridQueryCancel;
import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
+import org.apache.ignite.internal.util.typedef.F;
import org.h2.jdbc.JdbcConnection;
import org.jetbrains.annotations.Nullable;
@@ -53,8 +54,8 @@ class ReduceQueryRun {
/** */
private final int pageSize;
- /** Can be either CacheException in case of error or AffinityTopologyVersion to retry if needed. */
- private final AtomicReference<Object> state = new AtomicReference<>();
+ /** */
+ private final AtomicReference<State> state = new AtomicReference<>();
/** Future controlling {@code SELECT FOR UPDATE} query execution. */
private final GridNearTxSelectForUpdateFuture selectForUpdateFut;
@@ -86,30 +87,59 @@ class ReduceQueryRun {
}
/**
- * @param o Fail state object.
+ * Set state on exception.
+ *
+ * @param err error.
+ * @param nodeId Node ID.
+ */
+ void setStateOnException(@Nullable UUID nodeId, CacheException err) {
+ setState0(new State(nodeId, err, null, null));
+ }
+
+ /**
+ * Set state on map node leave.
+ *
+ * @param nodeId Node ID.
+ * @param topVer Topology version.
+ */
+ void setStateOnNodeLeave(UUID nodeId, AffinityTopologyVersion topVer) {
+ setState0(new State(nodeId, null, topVer, "Data node has left the grid during query execution [nodeId=" +
+ nodeId + ']'));
+ }
+
+ /**
+ * Set state on retry due to mapping failure.
+ *
* @param nodeId Node ID.
+ * @param topVer Topology version.
+ * @param retryCause Retry cause.
*/
- void state(Object o, @Nullable UUID nodeId) {
- assert o != null;
- assert o instanceof CacheException || o instanceof AffinityTopologyVersion : o.getClass();
+ void setStateOnRetry(UUID nodeId, AffinityTopologyVersion topVer, String retryCause) {
+ assert !F.isEmpty(retryCause);
- if (!state.compareAndSet(null, o))
+ setState0(new State(nodeId, null, topVer, retryCause));
+ }
+
+ /**
+ *
+ * @param state state
+ */
+ private void setState0(State state){
+ if (!this.state.compareAndSet(null, state))
return;
while (latch.getCount() != 0) // We don't need to wait for all nodes to reply.
latch.countDown();
- CacheException e = o instanceof CacheException ? (CacheException) o : null;
-
for (GridMergeIndex idx : idxs) // Fail all merge indexes.
- idx.fail(nodeId, e);
+ idx.fail(state.nodeId, state.ex);
}
/**
* @param e Error.
*/
void disconnected(CacheException e) {
- state(e, null);
+ setStateOnException(null, e);
}
/**
@@ -133,11 +163,45 @@ class ReduceQueryRun {
return conn;
}
+ /** */
+ boolean hasErrorOrRetry(){
+ return state.get() != null;
+ }
+
/**
- * @return State.
+ * @return Exception.
*/
- Object state() {
- return state.get();
+ CacheException exception() {
+ State st = state.get();
+
+ return st != null ? st.ex : null;
+ }
+
+ /**
+ * @return Retry topology version.
+ */
+ AffinityTopologyVersion retryTopologyVersion(){
+ State st = state.get();
+
+ return st != null ? st.retryTopVer : null;
+ }
+
+ /**
+ * @return Retry bode ID.
+ */
+ UUID retryNodeId() {
+ State st = state.get();
+
+ return st != null ? st.nodeId : null;
+ }
+
+ /**
+ * @return Retry cause.
+ */
+ String retryCause(){
+ State st = state.get();
+
+ return st != null ? st.retryCause : null;
}
/**
@@ -167,4 +231,29 @@ class ReduceQueryRun {
@Nullable public GridNearTxSelectForUpdateFuture selectForUpdateFuture() {
return selectForUpdateFut;
}
+
+ /**
+ * Error state.
+ */
+ private static class State {
+ /** Affected node (may be null in case of local node failure). */
+ private final UUID nodeId;
+
+ /** Error. */
+ private final CacheException ex;
+
+ /** Retry topology version. */
+ private final AffinityTopologyVersion retryTopVer;
+
+ /** Retry cause. */
+ private final String retryCause;
+
+ /** */
+ private State(UUID nodeId, CacheException ex, AffinityTopologyVersion retryTopVer, String retryCause){
+ this.nodeId = nodeId;
+ this.ex = ex;
+ this.retryTopVer = retryTopVer;
+ this.retryCause = retryCause;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/525a7754/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheCauseRetryMessageSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheCauseRetryMessageSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheCauseRetryMessageSelfTest.java
new file mode 100644
index 0000000..8c4358a
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheCauseRetryMessageSelfTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import javax.cache.CacheException;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryCancelRequest;
+import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.Organization;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.Person;
+
+/**
+ * Failed to reserve partitions for query (cache is not found on local node) Root cause test
+ */
+public class DisappearedCacheCauseRetryMessageSelfTest extends GridCommonAbstractTest {
+ /** */
+ private static final int NODES_COUNT = 2;
+ /** */
+ private static final String ORG = "org";
+ /** */
+ private IgniteCache<String, JoinSqlTestHelper.Person> personCache;
+ /** */
+ private IgniteCache<String, JoinSqlTestHelper.Organization> orgCache;
+
+ /** */
+ public void testDisappearedCacheCauseRetryMessage() {
+
+ SqlQuery<String, JoinSqlTestHelper.Person> qry = new SqlQuery<String, JoinSqlTestHelper.Person>(JoinSqlTestHelper.Person.class, JoinSqlTestHelper.JOIN_SQL).setArgs("Organization #0");
+
+ qry.setDistributedJoins(true);
+
+ try {
+ personCache.query(qry).getAll();
+
+ fail("No CacheException emitted.");
+ }
+ catch (CacheException e) {
+ assertTrue(e.getMessage(), e.getMessage().contains("Failed to reserve partitions for query (cache is not found on local node) ["));
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setCommunicationSpi(new TcpCommunicationSpi(){
+
+ volatile long reqId = -1;
+ /** {@inheritDoc} */
+ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) {
+ assert msg != null;
+
+ if ( GridIoMessage.class.isAssignableFrom(msg.getClass())){
+ GridIoMessage gridMsg = (GridIoMessage)msg;
+
+ if ( GridH2QueryRequest.class.isAssignableFrom( gridMsg.message().getClass() ) ){
+ GridH2QueryRequest req = (GridH2QueryRequest) (gridMsg.message());
+ reqId = req.requestId();
+ orgCache.destroy();
+ }
+ else if ( GridQueryCancelRequest.class.isAssignableFrom( gridMsg.message().getClass() ) ){
+ GridQueryCancelRequest req = (GridQueryCancelRequest) (gridMsg.message());
+
+ if (reqId == req.queryRequestId())
+ orgCache = DisappearedCacheCauseRetryMessageSelfTest.this.ignite(0).getOrCreateCache(new CacheConfiguration<String, Organization>(ORG)
+ .setCacheMode(CacheMode.REPLICATED)
+ .setIndexedTypes(String.class, JoinSqlTestHelper.Organization.class)
+ );
+
+ }
+ }
+
+ super.sendMessage(node, msg, ackC);
+ }
+ });
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ System.setProperty(IGNITE_SQL_RETRY_TIMEOUT, "5000");
+
+ startGridsMultiThreaded(NODES_COUNT, false);
+
+ personCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, Person>("pers")
+ .setIndexedTypes(String.class, JoinSqlTestHelper.Person.class)
+ );
+
+ orgCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, Organization>(ORG)
+ .setCacheMode(CacheMode.REPLICATED)
+ .setIndexedTypes(String.class, JoinSqlTestHelper.Organization.class)
+ );
+
+ awaitPartitionMapExchange();
+
+ JoinSqlTestHelper.populateDataIntoOrg(orgCache);
+
+ JoinSqlTestHelper.populateDataIntoPerson(personCache);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/525a7754/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheWasNotFoundMessageSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheWasNotFoundMessageSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheWasNotFoundMessageSelfTest.java
new file mode 100644
index 0000000..9928ed6
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheWasNotFoundMessageSelfTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.Organization;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.Person;
+
+/**
+ * Grid cache context is not registered for cache id root cause message test
+ */
+public class DisappearedCacheWasNotFoundMessageSelfTest extends GridCommonAbstractTest {
+ /** */
+ private static final int NODES_COUNT = 2;
+ /** */
+ private static final String ORG = "org";
+ /** */
+ private IgniteCache<String, JoinSqlTestHelper.Person> personCache;
+ /** */
+ private IgniteCache<String, JoinSqlTestHelper.Organization> orgCache;
+
+ /** */
+ public void testDisappearedCacheWasNotFoundMessage() {
+ SqlQuery<String, Person> qry = new SqlQuery<String, Person>(Person.class, JoinSqlTestHelper.JOIN_SQL).setArgs("Organization #0");
+
+ qry.setDistributedJoins(true);
+
+ try {
+ personCache.query(qry).getAll();
+
+ fail("No CacheException emitted.");
+ }
+ catch (CacheException e) {
+ assertTrue(e.getMessage(), e.getMessage().contains("Cache not found on local node"));
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setCommunicationSpi(new TcpCommunicationSpi(){
+ /** {@inheritDoc} */
+ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) {
+ assert msg != null;
+
+ if ( GridIoMessage.class.isAssignableFrom(msg.getClass())){
+ GridIoMessage gridMsg = (GridIoMessage)msg;
+
+ if ( GridH2QueryRequest.class.isAssignableFrom( gridMsg.message().getClass() ) ){
+ GridH2QueryRequest req = (GridH2QueryRequest) (gridMsg.message());
+
+ req.requestId();
+
+ orgCache.destroy();
+ }
+ }
+
+ super.sendMessage(node, msg, ackC);
+ }
+ });
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ System.setProperty(IGNITE_SQL_RETRY_TIMEOUT, "5000");
+
+ startGridsMultiThreaded(NODES_COUNT, false);
+
+ personCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, Person>("pers")
+ .setIndexedTypes(String.class, JoinSqlTestHelper.Person.class)
+ );
+
+ orgCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, Organization>(ORG)
+ .setCacheMode(CacheMode.REPLICATED)
+ .setIndexedTypes(String.class, Organization.class)
+ );
+
+ awaitPartitionMapExchange();
+
+ JoinSqlTestHelper.populateDataIntoOrg(orgCache);
+
+ JoinSqlTestHelper.populateDataIntoPerson(personCache);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/525a7754/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinSqlTestHelper.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinSqlTestHelper.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinSqlTestHelper.java
new file mode 100644
index 0000000..fe7821a
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinSqlTestHelper.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+
+/**
+ * Join sql test helper
+ */
+public class JoinSqlTestHelper {
+ /** */
+ private static final int ORG_COUNT = 100;
+
+ /** */
+ private static final int PERSON_PER_ORG_COUNT = 10;
+
+ /** */
+ static final String JOIN_SQL = "select * from Person, \"org\".Organization as org " +
+ "where Person.orgId = org.id " +
+ "and lower(org.name) = lower(?)";
+
+ /**
+ * Populate organization cache with test data
+ * @param cache @{IgniteCache}
+ */
+ static void populateDataIntoOrg(IgniteCache<String, Organization> cache) {
+ for (int i = 0; i < ORG_COUNT; i++) {
+ Organization org = new Organization();
+
+ org.setId("org" + i);
+
+ org.setName("Organization #" + i);
+
+ cache.put(org.getId(), org);
+ }
+ }
+
+ /**
+ * Populate person cache with test data
+ * @param cache @{IgniteCache}
+ */
+ static void populateDataIntoPerson(IgniteCache<String, Person> cache) {
+ int personId = 0;
+
+ for (int i = 0; i < ORG_COUNT; i++) {
+ Organization org = new Organization();
+
+ org.setId("org" + i);
+
+ org.setName("Organization #" + i);
+
+ for (int j = 0; j < PERSON_PER_ORG_COUNT; j++) {
+ Person prsn = new Person();
+
+ prsn.setId("pers" + personId);
+
+ prsn.setOrgId(org.getId());
+
+ prsn.setName("Person name #" + personId);
+
+ cache.put(prsn.getId(), prsn);
+
+ personId++;
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ public static class Person {
+ /** */
+ @QuerySqlField(index = true)
+ private String id;
+
+ /** */
+ @QuerySqlField(index = true)
+ private String orgId;
+
+ /** */
+ @QuerySqlField(index = true)
+ private String name;
+
+ /** */
+ public String getId() {
+ return id;
+ }
+
+ /** */
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /** */
+ public String getOrgId() {
+ return orgId;
+ }
+
+ /** */
+ public void setOrgId(String orgId) {
+ this.orgId = orgId;
+ }
+
+ /** */
+ public String getName() {
+ return name;
+ }
+
+ /** */
+ public void setName(String name) {
+ this.name = name;
+ }
+ }
+
+ /**
+ *
+ */
+ public static class Organization {
+ /** */
+ @QuerySqlField(index = true)
+ private String id;
+
+ /** */
+ @QuerySqlField(index = true)
+ private String name;
+
+ /** */
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /** */
+ public String getId() {
+ return id;
+ }
+
+ /** */
+ public String getName() {
+ return name;
+ }
+
+ /** */
+ public void setName(String name) {
+ this.name = name;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/525a7754/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/NonCollocatedRetryMessageSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/NonCollocatedRetryMessageSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/NonCollocatedRetryMessageSelfTest.java
new file mode 100644
index 0000000..c602225
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/NonCollocatedRetryMessageSelfTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import java.util.List;
+import javax.cache.Cache;
+import javax.cache.CacheException;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT;
+
+/**
+ * Failed to execute non-collocated query root cause message test
+ */
+public class NonCollocatedRetryMessageSelfTest extends GridCommonAbstractTest {
+ /** */
+ private static final int NODES_COUNT = 3;
+
+ /** */
+ private static final String ORG = "org";
+
+ /** */
+ private IgniteCache<String, JoinSqlTestHelper.Person> personCache;
+
+ /** */
+ public void testNonCollocatedRetryMessage() {
+ SqlQuery<String, JoinSqlTestHelper.Person> qry = new SqlQuery<String, JoinSqlTestHelper.Person>(JoinSqlTestHelper.Person.class, JoinSqlTestHelper.JOIN_SQL).setArgs("Organization #0");
+
+ qry.setDistributedJoins(true);
+
+ try {
+ List<Cache.Entry<String,JoinSqlTestHelper.Person>> prsns = personCache.query(qry).getAll();
+ fail("No CacheException emitted. Collection size="+prsns.size());
+ }
+ catch (CacheException e) {
+ assertTrue(e.getMessage(), e.getMessage().contains("Failed to execute non-collocated query"));
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setCommunicationSpi(new TcpCommunicationSpi(){
+ volatile long reqId = -1;
+ /** {@inheritDoc} */
+ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) {
+ assert msg != null;
+
+ if ( GridIoMessage.class.isAssignableFrom(msg.getClass())){
+ GridIoMessage gridMsg = (GridIoMessage)msg;
+
+ if ( GridH2QueryRequest.class.isAssignableFrom( gridMsg.message().getClass() ) ){
+ GridH2QueryRequest req = (GridH2QueryRequest) (gridMsg.message());
+
+ if (reqId < 0) {
+ reqId = req.requestId();
+
+ String shutName = getTestIgniteInstanceName(1);
+
+ stopGrid(shutName, true, false);
+ }
+ else if( reqId != req.requestId() ){
+ try {
+ U.sleep(IgniteSystemProperties.getLong(IGNITE_SQL_RETRY_TIMEOUT, GridReduceQueryExecutor.DFLT_RETRY_TIMEOUT));
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ // no-op
+ }
+ }
+ }
+ }
+ super.sendMessage(node, msg, ackC);
+ }
+ });
+
+ cfg.setDiscoverySpi(new TcpDiscoverySpi(){
+ public long getNodesJoined() {
+ return stats.joinedNodesCount();
+ }
+ });
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ System.setProperty(IGNITE_SQL_RETRY_TIMEOUT, "5000");
+
+ startGridsMultiThreaded(NODES_COUNT, false);
+
+ personCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, JoinSqlTestHelper.Person>("pers")
+ .setBackups(1)
+ .setIndexedTypes(String.class, JoinSqlTestHelper.Person.class)
+ );
+
+ final IgniteCache<String, JoinSqlTestHelper.Organization> orgCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, JoinSqlTestHelper.Organization>(ORG)
+ .setBackups(1)
+ .setIndexedTypes(String.class, JoinSqlTestHelper.Organization.class)
+ );
+
+ awaitPartitionMapExchange();
+
+ JoinSqlTestHelper.populateDataIntoOrg(orgCache);
+
+ JoinSqlTestHelper.populateDataIntoPerson(personCache);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+
+
+}
+
http://git-wip-us.apache.org/repos/asf/ignite/blob/525a7754/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
new file mode 100644
index 0000000..3269887
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import java.util.UUID;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2RetryException;
+import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest;
+import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT;
+import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.JOIN_SQL;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.Organization;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.Person;
+
+/**
+ * Test for 6 retry cases
+ */
+public class RetryCauseMessageSelfTest extends GridCommonAbstractTest {
+ /** */
+ private static final int NODES_COUNT = 2;
+
+ /** */
+ private static final String ORG_SQL = "select * from Organization";
+
+ /** */
+ private static final String ORG = "org";
+
+ /** */
+ private IgniteCache<String, Person> personCache;
+
+ /** */
+ private IgniteCache<String, Organization> orgCache;
+
+ /** */
+ private IgniteH2Indexing h2Idx;
+
+ /** */
+ @Override protected long getTestTimeout() {
+ return 600 * 1000;
+ }
+
+ /**
+ * Failed to reserve partitions for query (cache is not found on local node)
+ */
+ public void testSynthCacheWasNotFoundMessage() {
+ GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec");
+
+ GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec",
+ new MockGridMapQueryExecutor(null) {
+ @Override public void onMessage(UUID nodeId, Object msg) {
+ if (GridH2QueryRequest.class.isAssignableFrom(msg.getClass())) {
+ GridH2QueryRequest qryReq = (GridH2QueryRequest)msg;
+
+ qryReq.caches().add(Integer.MAX_VALUE);
+
+ startedExecutor.onMessage(nodeId, msg);
+
+ qryReq.caches().remove(qryReq.caches().size() - 1);
+ }
+ else
+ startedExecutor.onMessage(nodeId, msg);
+ }
+ }.insertRealExecutor(mapQryExec));
+
+ SqlQuery<String, Person> qry = new SqlQuery<String, Person>(Person.class, JOIN_SQL).setArgs("Organization #0");
+
+ qry.setDistributedJoins(true);
+
+ try {
+ personCache.query(qry).getAll();
+ }
+ catch (CacheException e) {
+ assertTrue(e.getMessage(), e.getMessage().contains("Failed to reserve partitions for query (cache is not found on local node) ["));
+
+ return;
+ }
+ finally {
+ GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec", mapQryExec);
+ }
+ fail();
+ }
+
+ /**
+ * Failed to reserve partitions for query (group reservation failed)
+ */
+ public void testGrpReservationFailureMessage() {
+ final GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec");
+
+ final ConcurrentMap<MapReservationKey, GridReservable> reservations = GridTestUtils.getFieldValue(mapQryExec, GridMapQueryExecutor.class, "reservations");
+
+ GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec",
+ new MockGridMapQueryExecutor(null) {
+ @Override public void onMessage(UUID nodeId, Object msg) {
+ if (GridH2QueryRequest.class.isAssignableFrom(msg.getClass())) {
+ final MapReservationKey grpKey = new MapReservationKey(ORG, null);
+
+ reservations.put(grpKey, new GridReservable() {
+
+ @Override public boolean reserve() {
+ return false;
+ }
+
+ @Override public void release() {}
+ });
+ }
+ startedExecutor.onMessage(nodeId, msg);
+ }
+ }.insertRealExecutor(mapQryExec));
+
+ SqlQuery<String, Person> qry = new SqlQuery<String, Person>(Person.class, JOIN_SQL).setArgs("Organization #0");
+
+ qry.setDistributedJoins(true);
+
+ try {
+ personCache.query(qry).getAll();
+ }
+ catch (CacheException e) {
+ assertTrue(e.getMessage().contains("Failed to reserve partitions for query (group reservation failed) ["));
+
+ return;
+ }
+ finally {
+ GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec", mapQryExec);
+ }
+ fail();
+ }
+
+ /**
+ * Failed to reserve partitions for query (partition of REPLICATED cache is not in OWNING state)
+ */
+ public void testReplicatedCacheReserveFailureMessage() {
+ GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec");
+
+ final GridKernalContext ctx = GridTestUtils.getFieldValue(mapQryExec, GridMapQueryExecutor.class, "ctx");
+
+ GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec",
+ new MockGridMapQueryExecutor(null) {
+ @Override public void onMessage(UUID nodeId, Object msg) {
+ if (GridH2QueryRequest.class.isAssignableFrom(msg.getClass())) {
+ GridH2QueryRequest qryReq = (GridH2QueryRequest)msg;
+
+ GridCacheContext<?, ?> cctx = ctx.cache().context().cacheContext(qryReq.caches().get(0));
+
+ GridDhtLocalPartition part = cctx.topology().localPartition(0, NONE, false);
+
+ AtomicLong aState = GridTestUtils.getFieldValue(part, GridDhtLocalPartition.class, "state");
+
+ long stateVal = aState.getAndSet(2);
+
+ startedExecutor.onMessage(nodeId, msg);
+
+ aState.getAndSet(stateVal);
+ }
+ else
+ startedExecutor.onMessage(nodeId, msg);
+ }
+ }.insertRealExecutor(mapQryExec));
+
+ SqlQuery<String, Organization> qry = new SqlQuery<>(Organization.class, ORG_SQL);
+
+ qry.setDistributedJoins(true);
+ try {
+ orgCache.query(qry).getAll();
+ }
+ catch (CacheException e) {
+ assertTrue(e.getMessage().contains("Failed to reserve partitions for query (partition of REPLICATED cache is not in OWNING state) ["));
+
+ return;
+ }
+ finally {
+ GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec", mapQryExec);
+ }
+ fail();
+ }
+
+ /**
+ * Failed to reserve partitions for query (partition of PARTITIONED cache cannot be reserved)
+ */
+ public void testPartitionedCacheReserveFailureMessage() {
+ GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec");
+
+ final GridKernalContext ctx = GridTestUtils.getFieldValue(mapQryExec, GridMapQueryExecutor.class, "ctx");
+
+ GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec",
+ new MockGridMapQueryExecutor(null) {
+ @Override public void onMessage(UUID nodeId, Object msg) {
+ if (GridH2QueryRequest.class.isAssignableFrom(msg.getClass())) {
+ GridH2QueryRequest qryReq = (GridH2QueryRequest)msg;
+
+ GridCacheContext<?, ?> cctx = ctx.cache().context().cacheContext(qryReq.caches().get(0));
+
+ GridDhtLocalPartition part = cctx.topology().localPartition(0, NONE, false);
+
+ AtomicLong aState = GridTestUtils.getFieldValue(part, GridDhtLocalPartition.class, "state");
+
+ long stateVal = aState.getAndSet(2);
+
+ startedExecutor.onMessage(nodeId, msg);
+
+ aState.getAndSet(stateVal);
+ }
+ else
+ startedExecutor.onMessage(nodeId, msg);
+
+ }
+ }.insertRealExecutor(mapQryExec));
+
+ SqlQuery<String, Person> qry = new SqlQuery<String, Person>(Person.class, JOIN_SQL).setArgs("Organization #0");
+
+ qry.setDistributedJoins(true);
+ try {
+ personCache.query(qry).getAll();
+ }
+ catch (CacheException e) {
+ assertTrue(e.getMessage().contains("Failed to reserve partitions for query (partition of PARTITIONED cache cannot be reserved) ["));
+
+ return;
+ }
+ finally {
+ GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec", mapQryExec);
+ }
+ fail();
+ }
+
+ /**
+ * Failed to execute non-collocated query (will retry)
+ */
+ public void testNonCollocatedFailureMessage() {
+ final GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec");
+
+ final ConcurrentMap<MapReservationKey, GridReservable> reservations = GridTestUtils.getFieldValue(mapQryExec, GridMapQueryExecutor.class, "reservations");
+
+ GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec",
+ new MockGridMapQueryExecutor(null) {
+ @Override public void onMessage(UUID nodeId, Object msg) {
+ if (GridH2QueryRequest.class.isAssignableFrom(msg.getClass())) {
+ final MapReservationKey grpKey = new MapReservationKey(ORG, null);
+
+ reservations.put(grpKey, new GridReservable() {
+
+ @Override public boolean reserve() {
+ throw new GridH2RetryException("test retry exception");
+ }
+
+ @Override public void release() {
+ }
+ });
+ }
+ startedExecutor.onMessage(nodeId, msg);
+
+ }
+ }.insertRealExecutor(mapQryExec));
+
+ SqlQuery<String, Person> qry = new SqlQuery<String, Person>(Person.class, JOIN_SQL).setArgs("Organization #0");
+
+ qry.setDistributedJoins(true);
+ try {
+ personCache.query(qry).getAll();
+ }
+ catch (CacheException e) {
+ assertTrue(e.getMessage().contains("Failed to execute non-collocated query (will retry) ["));
+
+ return;
+ }
+ finally {
+ GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec", mapQryExec);
+ }
+ fail();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setCommunicationSpi(new TcpCommunicationSpi(){
+ /** {@inheritDoc} */
+ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) {
+ assert msg != null;
+
+ super.sendMessage(node, msg, ackC);
+ }
+ });
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ System.setProperty(IGNITE_SQL_RETRY_TIMEOUT, "5000");
+
+ Ignite ignite = startGridsMultiThreaded(NODES_COUNT, false);
+
+ GridQueryProcessor qryProc = grid(ignite.name()).context().query();
+
+ h2Idx = GridTestUtils.getFieldValue(qryProc, GridQueryProcessor.class, "idx");
+
+ personCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, Person>("pers")
+ .setIndexedTypes(String.class, Person.class)
+ );
+
+ orgCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, Organization>(ORG)
+ .setCacheMode(CacheMode.REPLICATED)
+ .setIndexedTypes(String.class, Organization.class)
+ );
+
+ awaitPartitionMapExchange();
+
+ JoinSqlTestHelper.populateDataIntoOrg(orgCache);
+
+ JoinSqlTestHelper.populateDataIntoPerson(personCache);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+
+
+ /**
+ * Wrapper around @{GridMapQueryExecutor}
+ */
+ private abstract static class MockGridMapQueryExecutor extends GridMapQueryExecutor {
+
+ /**
+ * Wrapped executor
+ */
+ GridMapQueryExecutor startedExecutor;
+
+ /** */
+ MockGridMapQueryExecutor insertRealExecutor(GridMapQueryExecutor realExecutor) {
+ this.startedExecutor = realExecutor;
+ return this;
+ }
+
+ /**
+ * @param busyLock Busy lock.
+ */
+ MockGridMapQueryExecutor(GridSpinBusyLock busyLock) {
+ super(busyLock);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onMessage(UUID nodeId, Object msg) {
+ startedExecutor.onMessage(nodeId, msg);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancelLazyWorkers() {
+ startedExecutor.cancelLazyWorkers();
+ }
+
+ /** {@inheritDoc} */
+ @Override GridSpinBusyLock busyLock() {
+ return startedExecutor.busyLock();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onCacheStop(String cacheName) {
+ startedExecutor.onCacheStop(cacheName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stopAndUnregisterCurrentLazyWorker() {
+ startedExecutor.stopAndUnregisterCurrentLazyWorker();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void unregisterLazyWorker(MapQueryLazyWorker worker) {
+ startedExecutor.unregisterLazyWorker(worker);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int registeredLazyWorkers() {
+ return startedExecutor.registeredLazyWorkers();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/525a7754/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
index 093423d..536834c 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
@@ -51,6 +51,10 @@ import org.apache.ignite.internal.processors.query.IgniteCacheGroupsSqlDistribut
import org.apache.ignite.internal.processors.query.IgniteCacheGroupsSqlSegmentedIndexMultiNodeSelfTest;
import org.apache.ignite.internal.processors.query.IgniteCacheGroupsSqlSegmentedIndexSelfTest;
import org.apache.ignite.internal.processors.query.h2.twostep.CacheQueryMemoryLeakTest;
+import org.apache.ignite.internal.processors.query.h2.twostep.DisappearedCacheCauseRetryMessageSelfTest;
+import org.apache.ignite.internal.processors.query.h2.twostep.DisappearedCacheWasNotFoundMessageSelfTest;
+import org.apache.ignite.internal.processors.query.h2.twostep.NonCollocatedRetryMessageSelfTest;
+import org.apache.ignite.internal.processors.query.h2.twostep.RetryCauseMessageSelfTest;
import org.apache.ignite.testframework.IgniteTestSuite;
/**
@@ -110,6 +114,11 @@ public class IgniteCacheQuerySelfTestSuite2 extends TestSuite {
suite.addTestSuite(CacheQueryMemoryLeakTest.class);
+ suite.addTestSuite(NonCollocatedRetryMessageSelfTest.class);
+ suite.addTestSuite(RetryCauseMessageSelfTest.class);
+ suite.addTestSuite(DisappearedCacheCauseRetryMessageSelfTest.class);
+ suite.addTestSuite(DisappearedCacheWasNotFoundMessageSelfTest.class);
+
return suite;
}
}
[29/50] [abbrv] ignite git commit: IGNITE-9408: Update Apache Mesos
version. - Fixes #4650.
Posted by ag...@apache.org.
IGNITE-9408: Update Apache Mesos version. - Fixes #4650.
Signed-off-by: shroman <rs...@yahoo.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5b4d5630
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5b4d5630
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5b4d5630
Branch: refs/heads/ignite-5960
Commit: 5b4d56301887cbcde844da5b4b9fe1bb348e0f86
Parents: 35706b9
Author: shroman <rs...@yahoo.com>
Authored: Wed Sep 5 16:24:07 2018 +0900
Committer: shroman <rs...@yahoo.com>
Committed: Wed Sep 5 16:24:07 2018 +0900
----------------------------------------------------------------------
modules/mesos/pom.xml | 2 +-
.../java/org/apache/ignite/mesos/IgniteFramework.java | 3 +--
.../org/apache/ignite/mesos/IgniteSchedulerSelfTest.java | 11 +++++++++++
3 files changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b4d5630/modules/mesos/pom.xml
----------------------------------------------------------------------
diff --git a/modules/mesos/pom.xml b/modules/mesos/pom.xml
index 5880259..a8c7d8b 100644
--- a/modules/mesos/pom.xml
+++ b/modules/mesos/pom.xml
@@ -35,7 +35,7 @@
<url>http://ignite.apache.org</url>
<properties>
- <mesos.version>0.22.0</mesos.version>
+ <mesos.version>1.5.0</mesos.version>
<ignite.latest.url>http://ignite.run/download_ignite.php</ignite.latest.url>
<ignite.direct.url>https://archive.apache.org/dist/ignite/%s/apache-ignite-fabric-%s-bin.zip</ignite.direct.url>
</properties>
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b4d5630/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteFramework.java
----------------------------------------------------------------------
diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteFramework.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteFramework.java
index eea510a..addd3e0 100644
--- a/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteFramework.java
+++ b/modules/mesos/src/main/java/org/apache/ignite/mesos/IgniteFramework.java
@@ -17,7 +17,6 @@
package org.apache.ignite.mesos;
-import com.google.protobuf.ByteString;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.ignite.mesos.resource.IgniteProvider;
@@ -105,7 +104,7 @@ public class IgniteFramework {
Protos.Credential cred = Protos.Credential.newBuilder()
.setPrincipal(System.getenv(DEFAULT_PRINCIPAL))
- .setSecret(ByteString.copyFrom(System.getenv(DEFAULT_SECRET).getBytes()))
+ .setSecret(System.getenv(DEFAULT_SECRET))
.build();
driver = new MesosSchedulerDriver(scheduler, igniteFramework.getFrameworkInfo(), clusterProps.masterUrl(),
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b4d5630/modules/mesos/src/test/java/org/apache/ignite/mesos/IgniteSchedulerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/mesos/src/test/java/org/apache/ignite/mesos/IgniteSchedulerSelfTest.java b/modules/mesos/src/test/java/org/apache/ignite/mesos/IgniteSchedulerSelfTest.java
index 099daa2..4e48569 100644
--- a/modules/mesos/src/test/java/org/apache/ignite/mesos/IgniteSchedulerSelfTest.java
+++ b/modules/mesos/src/test/java/org/apache/ignite/mesos/IgniteSchedulerSelfTest.java
@@ -466,6 +466,12 @@ public class IgniteSchedulerSelfTest extends TestCase {
}
/** {@inheritDoc} */
+ @Override public Protos.Status acceptOffers(Collection<Protos.OfferID> collection,
+ Collection<Protos.Offer.Operation> collection1, Protos.Filters filters) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Override public Protos.Status declineOffer(Protos.OfferID offerId, Protos.Filters filters) {
declinedOffer = offerId;
@@ -485,6 +491,11 @@ public class IgniteSchedulerSelfTest extends TestCase {
}
/** {@inheritDoc} */
+ @Override public Protos.Status suppressOffers() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Override public Protos.Status acknowledgeStatusUpdate(Protos.TaskStatus status) {
return null;
}
[40/50] [abbrv] ignite git commit: IGNITE-8915: Check moved to
GridQueryProcessor. Check for SqlFieldsQuery added. - Fixes #4414.
Posted by ag...@apache.org.
IGNITE-8915: Check moved to GridQueryProcessor. Check for SqlFieldsQuery added. - Fixes #4414.
Signed-off-by: Nikolay Izhikov <ni...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/445d375a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/445d375a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/445d375a
Branch: refs/heads/ignite-5960
Commit: 445d375ac31c64e9f7896f578b96fca43095c6be
Parents: a549a74
Author: Nikolay Izhikov <ni...@apache.org>
Authored: Thu Sep 6 13:05:39 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Thu Sep 6 13:05:39 2018 +0300
----------------------------------------------------------------------
.../processors/cache/IgniteCacheProxyImpl.java | 3 -
.../processors/query/GridQueryProcessor.java | 24 +++++++-
.../cache/IgniteCacheAbstractQuerySelfTest.java | 54 ++++++++++++++++-
.../IgniteCacheReplicatedQuerySelfTest.java | 27 ---------
.../index/H2DynamicIndexAbstractSelfTest.java | 6 ++
.../local/IgniteCacheLocalQuerySelfTest.java | 64 +++++++++++++++++++-
6 files changed, 144 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/445d375a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index 69ea562..225fa81 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -760,9 +760,6 @@ public class IgniteCacheProxyImpl<K, V> extends AsyncSupportAdapter<IgniteCache<
(qry instanceof SqlQuery || qry instanceof SqlFieldsQuery || qry instanceof TextQuery))
throw new CacheException("Failed to execute query. Add module 'ignite-indexing' to the classpath " +
"of all Ignite nodes.");
-
- if (qry.isLocal() && (qry instanceof SqlQuery) && ctx.kernalContext().clientNode())
- throw new CacheException("Execution of local sql query on client node disallowed.");
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/445d375a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index bd5d91c..eb3f2a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -2111,7 +2111,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
final boolean failOnMultipleStmts) {
checkxEnabled();
- validateSqlFieldsQuery(qry);
+ validateSqlFieldsQuery(qry, ctx, cctx);
if (!ctx.state().publicApiActiveState(true)) {
throw new IgniteException("Can not perform the operation because the cluster is inactive. Note, that " +
@@ -2161,13 +2161,31 @@ public class GridQueryProcessor extends GridProcessorAdapter {
* Validate SQL fields query.
*
* @param qry Query.
+ * @param ctx Kernal context.
+ * @param cctx Cache context.
*/
- private static void validateSqlFieldsQuery(SqlFieldsQuery qry) {
+ private static void validateSqlFieldsQuery(SqlFieldsQuery qry, GridKernalContext ctx,
+ @Nullable GridCacheContext<?, ?> cctx) {
if (qry.isReplicatedOnly() && qry.getPartitions() != null)
throw new CacheException("Partitions are not supported in replicated only mode.");
if (qry.isDistributedJoins() && qry.getPartitions() != null)
throw new CacheException("Using both partitions and distributed JOINs is not supported for the same query");
+
+ if (qry.isLocal() && ctx.clientNode() && (cctx == null || cctx.config().getCacheMode() != CacheMode.LOCAL))
+ throw new CacheException("Execution of local SqlFieldsQuery on client node disallowed.");
+ }
+
+ /**
+ * Validate SQL query.
+ *
+ * @param qry Query.
+ * @param ctx Kernal context.
+ * @param cctx Cache context.
+ */
+ private static void validateSqlQuery(SqlQuery qry, GridKernalContext ctx, GridCacheContext<?, ?> cctx) {
+ if (qry.isLocal() && ctx.clientNode() && cctx.config().getCacheMode() != CacheMode.LOCAL)
+ throw new CacheException("Execution of local SqlQuery on client node disallowed.");
}
/**
@@ -2238,6 +2256,8 @@ public class GridQueryProcessor extends GridProcessorAdapter {
*/
public <K, V> QueryCursor<Cache.Entry<K,V>> querySql(final GridCacheContext<?,?> cctx, final SqlQuery qry,
boolean keepBinary) {
+ validateSqlQuery(qry, ctx, cctx);
+
if (qry.isReplicatedOnly() && qry.getPartitions() != null)
throw new CacheException("Partitions are not supported in replicated only mode.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/445d375a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
index a845aaa..ac9de6f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
@@ -64,6 +64,7 @@ import org.apache.ignite.cache.query.annotations.QueryTextField;
import org.apache.ignite.cache.store.CacheStore;
import org.apache.ignite.cache.store.CacheStoreAdapter;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.NearCacheConfiguration;
import org.apache.ignite.events.CacheQueryExecutedEvent;
@@ -96,6 +97,7 @@ import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED;
import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_OBJECT_READ;
import static org.apache.ignite.internal.processors.cache.query.CacheQueryType.FULL_TEXT;
import static org.apache.ignite.internal.processors.cache.query.CacheQueryType.SCAN;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
import static org.junit.Assert.assertArrayEquals;
/**
@@ -142,9 +144,12 @@ public abstract class IgniteCacheAbstractQuerySelfTest extends GridCommonAbstrac
c.setDiscoverySpi(new TcpDiscoverySpi().setForceServerMode(true).setIpFinder(ipFinder));
- if (igniteInstanceName.startsWith("client"))
+ if (igniteInstanceName.startsWith("client")) {
c.setClientMode(true);
+ c.setDataStorageConfiguration(new DataStorageConfiguration());
+ }
+
return c;
}
@@ -1774,6 +1779,53 @@ public abstract class IgniteCacheAbstractQuerySelfTest extends GridCommonAbstrac
}
/**
+ * @throws Exception If failed.
+ */
+ public void testLocalSqlQueryFromClient() throws Exception {
+ try {
+ Ignite g = startGrid("client");
+
+ IgniteCache<Integer, Integer> c = jcache(g, Integer.class, Integer.class);
+
+ for (int i = 0; i < 10; i++)
+ c.put(i, i);
+
+ SqlQuery<Integer, Integer> qry = new SqlQuery<>(Integer.class, "_key >= 5 order by _key");
+
+ qry.setLocal(true);
+
+ assertThrowsWithCause(() -> c.query(qry), CacheException.class);
+ }
+ finally {
+ stopGrid("client");
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testLocalSqlFieldsQueryFromClient() throws Exception {
+ try {
+ Ignite g = startGrid("client");
+
+ IgniteCache<UUID, Person> c = jcache(g, UUID.class, Person.class);
+
+ Person p = new Person("Jon", 1500);
+
+ c.put(p.id(), p);
+
+ SqlFieldsQuery qry = new SqlFieldsQuery("select count(*) from Person");
+
+ qry.setLocal(true);
+
+ assertThrowsWithCause(() -> c.query(qry), CacheException.class);
+ }
+ finally {
+ stopGrid("client");
+ }
+ }
+
+ /**
*
*/
private static class ArrayObject implements Serializable {
http://git-wip-us.apache.org/repos/asf/ignite/blob/445d375a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
index bd3dffd..13942c2 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
@@ -28,7 +28,6 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import javax.cache.Cache;
-import javax.cache.CacheException;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteException;
@@ -53,7 +52,6 @@ import org.apache.ignite.transactions.Transaction;
import static org.apache.ignite.cache.CacheMode.REPLICATED;
import static org.apache.ignite.cache.CachePeekMode.ALL;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
-import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
/**
* Tests replicated query.
@@ -163,31 +161,6 @@ public class IgniteCacheReplicatedQuerySelfTest extends IgniteCacheAbstractQuery
}
/**
- * @throws Exception If failed.
- */
- public void testClientsLocalQuery() throws Exception {
- try {
- Ignite g = startGrid("client");
-
- IgniteCache<Integer, Integer> c = jcache(g, Integer.class, Integer.class);
-
- for (int i = 0; i < 10; i++)
- c.put(i, i);
-
- assertEquals(0, c.localSize());
-
- SqlQuery<Integer, Integer> qry = new SqlQuery<>(Integer.class, "_key >= 5 order by _key");
-
- qry.setLocal(true);
-
- assertThrowsWithCause(() -> c.query(qry), CacheException.class);
- }
- finally {
- stopGrid("client");
- }
- }
-
- /**
* JUnit.
*
* @throws Exception If failed.
http://git-wip-us.apache.org/repos/asf/ignite/blob/445d375a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
index 9579b79..10ef56f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
@@ -85,6 +85,9 @@ public abstract class H2DynamicIndexAbstractSelfTest extends AbstractSchemaSelfT
// Test that local queries on all nodes use new index.
for (int i = 0 ; i < 4; i++) {
+ if (ignite(i).configuration().isClientMode())
+ continue;
+
List<List<?>> locRes = ignite(i).cache("cache").query(new SqlFieldsQuery("explain select \"id\" from " +
"\"cache\".\"ValueClass\" where \"field1\" = 'A'").setLocal(true)).getAll();
@@ -155,6 +158,9 @@ public abstract class H2DynamicIndexAbstractSelfTest extends AbstractSchemaSelfT
// Test that no local queries on all nodes use new index.
for (int i = 0 ; i < 4; i++) {
+ if (ignite(i).configuration().isClientMode())
+ continue;
+
List<List<?>> locRes = ignite(i).cache("cache").query(new SqlFieldsQuery("explain select \"id\" from " +
"\"cache\".\"ValueClass\" where \"field1\" = 'A'").setLocal(true)).getAll();
http://git-wip-us.apache.org/repos/asf/ignite/blob/445d375a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java
index 2570bc8..2272f27 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java
@@ -19,9 +19,12 @@ package org.apache.ignite.internal.processors.cache.local;
import java.util.Iterator;
import java.util.List;
+import java.util.UUID;
import javax.cache.Cache;
+import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.SqlQuery;
@@ -93,4 +96,63 @@ public class IgniteCacheLocalQuerySelfTest extends IgniteCacheAbstractQuerySelfT
cache.destroy();
}
}
-}
\ No newline at end of file
+
+ /** {@inheritDoc} */
+ @Override public void testLocalSqlQueryFromClient() throws Exception {
+ try {
+ Ignite g = startGrid("client");
+
+ IgniteCache<Integer, Integer> c = jcache(g, Integer.class, Integer.class);
+
+ for (int i = 0; i < 10; i++)
+ c.put(i, i);
+
+ SqlQuery<Integer, Integer> qry = new SqlQuery<>(Integer.class, "_key >= 5 order by _key");
+
+ qry.setLocal(true);
+
+ try(QueryCursor<Cache.Entry<Integer, Integer>> qryCursor = c.query(qry)) {
+ assertNotNull(qryCursor);
+
+ List<Cache.Entry<Integer, Integer>> res = qryCursor.getAll();
+
+ assertNotNull(res);
+
+ assertEquals(5, res.size());
+ }
+ }
+ finally {
+ stopGrid("client");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testLocalSqlFieldsQueryFromClient() throws Exception {
+ try {
+ Ignite g = startGrid("client");
+
+ IgniteCache<UUID, Person> c = jcache(g, UUID.class, Person.class);
+
+ Person p = new Person("Jon", 1500);
+
+ c.put(p.id(), p);
+
+ SqlFieldsQuery qry = new SqlFieldsQuery("select * from Person");
+
+ qry.setLocal(true);
+
+ try(FieldsQueryCursor<List<?>> qryCursor = c.query(qry)) {
+ assertNotNull(qryCursor);
+
+ List<List<?>> res = qryCursor.getAll();
+
+ assertNotNull(res);
+
+ assertEquals(1, res.size());
+ }
+ }
+ finally {
+ stopGrid("client");
+ }
+ }
+}
[32/50] [abbrv] ignite git commit: IGNITE-9402 Throws exception if
WALWriter failed during flush buffer. - Fixes #4640.
Posted by ag...@apache.org.
IGNITE-9402 Throws exception if WALWriter failed during flush buffer. - Fixes #4640.
Signed-off-by: Dmitriy Govorukhin <dm...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1fd72477
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1fd72477
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1fd72477
Branch: refs/heads/ignite-5960
Commit: 1fd72477e336bc48b2568d58114a2d15f5036d12
Parents: ef4a02d
Author: Anton Kalashnikov <ka...@yandex.ru>
Authored: Wed Sep 5 14:43:47 2018 +0300
Committer: Dmitriy Govorukhin <dm...@gmail.com>
Committed: Wed Sep 5 14:43:47 2018 +0300
----------------------------------------------------------------------
.../wal/FileWriteAheadLogManager.java | 22 +++++++++++---------
.../IgnitePdsWithIndexingCoreTestSuite.java | 2 ++
2 files changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1fd72477/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index eb5ae95..634cab3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -78,7 +78,6 @@ import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.persistence.StorageException;
import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
@@ -90,12 +89,12 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter
import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
import org.apache.ignite.internal.processors.cache.persistence.DataStorageMetricsImpl;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
-import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator.AbstractFileDescriptor;
import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
@@ -2682,7 +2681,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
*
* @param ptr Pointer.
*/
- private void flushOrWait(FileWALPointer ptr) {
+ private void flushOrWait(FileWALPointer ptr) throws IgniteCheckedException {
if (ptr != null) {
// If requested obsolete file index, it must be already flushed by close.
if (ptr.index() != idx)
@@ -2695,7 +2694,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/**
* @param ptr Pointer.
*/
- private void flush(FileWALPointer ptr) {
+ private void flush(FileWALPointer ptr) throws IgniteCheckedException {
if (ptr == null) { // Unconditional flush.
walWriter.flushAll();
@@ -3275,8 +3274,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/** {@inheritDoc} */
@Override protected void body() {
- Throwable err = null;
-
try {
while (!isCancelled()) {
while (waiters.isEmpty()) {
@@ -3399,21 +3396,21 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/**
* Forces all made changes to the file.
*/
- void force() {
+ void force() throws IgniteCheckedException {
flushBuffer(FILE_FORCE);
}
/**
* Closes file.
*/
- void close() {
+ void close() throws IgniteCheckedException {
flushBuffer(FILE_CLOSE);
}
/**
* Flushes all data from the buffer.
*/
- void flushAll() {
+ void flushAll() throws IgniteCheckedException {
flushBuffer(UNCONDITIONAL_FLUSH);
}
@@ -3421,7 +3418,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
* @param expPos Expected position.
*/
@SuppressWarnings("ForLoopReplaceableByForEach")
- void flushBuffer(long expPos) {
+ void flushBuffer(long expPos) throws IgniteCheckedException {
if (mmap)
return;
@@ -3447,6 +3444,11 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (val == Long.MIN_VALUE) {
waiters.remove(t);
+ Throwable walWriterError = walWriter.err;
+
+ if (walWriterError != null)
+ throw new IgniteCheckedException("Flush buffer failed.", walWriterError);
+
return;
}
else
http://git-wip-us.apache.org/repos/asf/ignite/blob/1fd72477/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index 491bab7..2989ccd 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -72,7 +72,9 @@ public class IgnitePdsWithIndexingCoreTestSuite extends TestSuite {
suite.addTestSuite(IgnitePdsTxHistoricalRebalancingTest.class);
suite.addTestSuite(IgniteWalRecoveryPPCTest.class);
+
suite.addTestSuite(IgnitePdsDiskErrorsRecoveringTest.class);
+
suite.addTestSuite(IgnitePdsCacheDestroyDuringCheckpointTest.class);
suite.addTestSuite(IgnitePdsBinaryMetadataOnClusterRestartTest.class);
[13/50] [abbrv] ignite git commit: IGNITE-8886: Binary: fixed footer
offset calculation when raw data is present. This closes #4643.
Posted by ag...@apache.org.
IGNITE-8886: Binary: fixed footer offset calculation when raw data is present. This closes #4643.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8cb4591d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8cb4591d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8cb4591d
Branch: refs/heads/ignite-5960
Commit: 8cb4591d691e9438fd0bf0809bb105229b4f7ff8
Parents: 4153b74
Author: Ilya Kasnacheev <il...@gmail.com>
Authored: Mon Sep 3 16:50:05 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Mon Sep 3 16:50:05 2018 +0300
----------------------------------------------------------------------
.../internal/binary/BinaryReaderExImpl.java | 2 +-
.../binary/BinaryMarshallerSelfTest.java | 35 ++++++++++++++++++--
2 files changed, 33 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8cb4591d/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
index ab1f874..38934f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
@@ -240,7 +240,7 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Bina
footerStart = start + offset;
if (BinaryUtils.hasRaw(flags)) {
- footerLen = len - offset - 4;
+ footerLen = len - offset;
rawOff = start + in.readIntPositioned(start + len - 4);
}
else {
http://git-wip-us.apache.org/repos/asf/ignite/blob/8cb4591d/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index 4a7d5d5..81a087d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -3064,6 +3064,33 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest {
/**
* @throws Exception If failed.
*/
+ public void testMixedRawCollections() throws Exception {
+ Collection<String> excludedClasses = Arrays.asList(
+ ObjectRaw.class.getName(),
+ ObjectWithRaw.class.getName(),
+ Value.class.getName());
+
+ BinaryMarshaller m0 = binaryMarshaller(null, excludedClasses);
+ BinaryMarshaller m1 = binaryMarshaller();
+
+ Value obj = new Value(27);
+ ObjectWithRaw objectWithRaw = new ObjectWithRaw(27, 13);
+ ObjectRaw objectRaw = new ObjectRaw(27, 13);
+
+ Value objOther = new Value(26);
+ ObjectWithRaw objectWithRawOther = new ObjectWithRaw(26, 13);
+ ObjectRaw objectRawOther = new ObjectRaw(26, 13);
+
+ ArrayList collection = new ArrayList(Arrays.asList(
+ obj, objectWithRawOther, objectRaw, objectWithRaw, objectRawOther, objOther));
+
+ marshalUnmarshal(collection, m0);
+ marshalUnmarshal(collection, m1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testBinaryEquals() throws Exception {
Collection<String> excludedClasses = Arrays.asList(
ObjectRaw.class.getName(),
@@ -5473,8 +5500,10 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
- val0 = reader.rawReader().readInt();
- val1 = reader.rawReader().readInt();
+ BinaryRawReader rawReader = reader.rawReader();
+
+ val0 = rawReader.readInt();
+ val1 = rawReader.readInt();
}
}
@@ -5592,4 +5621,4 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest {
}
}
}
-}
\ No newline at end of file
+}
[37/50] [abbrv] ignite git commit: IGNITE-8936: Remove
AffinityAssignment#clientEventChange as not used. - Fixes #4448.
Posted by ag...@apache.org.
IGNITE-8936: Remove AffinityAssignment#clientEventChange as not used. - Fixes #4448.
Signed-off-by: Nikolay Izhikov <ni...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7f22f08f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7f22f08f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7f22f08f
Branch: refs/heads/ignite-5960
Commit: 7f22f08f7db0f45a5cde97307c68fc81cb6eb26c
Parents: 6225c56
Author: Nikolai Kulagin <31...@users.noreply.github.com>
Authored: Thu Sep 6 12:43:08 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Thu Sep 6 12:43:08 2018 +0300
----------------------------------------------------------------------
.../processors/affinity/AffinityAssignment.java | 6 ------
.../processors/affinity/GridAffinityAssignment.java | 15 ---------------
.../affinity/HistoryAffinityAssignment.java | 9 ---------
3 files changed, 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7f22f08f/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java
index 1c321d3..eb06d3e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java
@@ -29,12 +29,6 @@ import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinator;
*/
public interface AffinityAssignment {
/**
- * @return {@code True} if related discovery event did not not cause affinity assignment change and
- * this assignment is just reference to the previous one.
- */
- public boolean clientEventChange();
-
- /**
* @return Affinity assignment computed by affinity function.
*/
public List<List<ClusterNode>> idealAssignment();
http://git-wip-us.apache.org/repos/asf/ignite/blob/7f22f08f/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
index f96bc9d..c4dfa8b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
@@ -65,9 +65,6 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable
/** */
private transient List<List<ClusterNode>> idealAssignment;
- /** */
- private final boolean clientEvtChange;
-
/**
* Constructs cached affinity calculations item.
*
@@ -78,7 +75,6 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable
primary = new HashMap<>();
backup = new HashMap<>();
mvccCrd = null;
- clientEvtChange = false;
}
/**
@@ -101,7 +97,6 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable
primary = new HashMap<>();
backup = new HashMap<>();
- clientEvtChange = false;
initPrimaryBackupMaps();
}
@@ -118,16 +113,6 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable
primary = aff.primary;
backup = aff.backup;
mvccCrd = aff.mvccCrd;
-
- clientEvtChange = true;
- }
-
- /**
- * @return {@code True} if related discovery event did not not cause affinity assignment change and
- * this assignment is just reference to the previous one.
- */
- public boolean clientEventChange() {
- return clientEvtChange;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/7f22f08f/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java
index 1a6b2ef..a949648 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java
@@ -43,9 +43,6 @@ public class HistoryAffinityAssignment implements AffinityAssignment {
private final List<List<ClusterNode>> idealAssignment;
/** */
- private final boolean clientEvtChange;
-
- /** */
private final MvccCoordinator mvccCrd;
/**
@@ -56,7 +53,6 @@ public class HistoryAffinityAssignment implements AffinityAssignment {
this.assignment = assign.assignment();
this.idealAssignment = assign.idealAssignment();
this.mvccCrd = assign.mvccCoordinator();
- this.clientEvtChange = assign.clientEventChange();
}
/** {@inheritDoc} */
@@ -65,11 +61,6 @@ public class HistoryAffinityAssignment implements AffinityAssignment {
}
/** {@inheritDoc} */
- @Override public boolean clientEventChange() {
- return clientEvtChange;
- }
-
- /** {@inheritDoc} */
@Override public List<List<ClusterNode>> idealAssignment() {
return idealAssignment;
}
[18/50] [abbrv] ignite git commit: Refactoring to improve constant
usage for file suffixes. - Fixes #4652.
Posted by ag...@apache.org.
Refactoring to improve constant usage for file suffixes. - Fixes #4652.
Signed-off-by: Dmitriy Govorukhin <dm...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6e7b86e8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6e7b86e8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6e7b86e8
Branch: refs/heads/ignite-5960
Commit: 6e7b86e8ed127904e3799144527c90ff5818a674
Parents: 525a775
Author: Pavel Voronkin <pv...@gridgain.com>
Authored: Tue Sep 4 12:41:14 2018 +0300
Committer: Dmitriy Govorukhin <dm...@gmail.com>
Committed: Tue Sep 4 12:41:14 2018 +0300
----------------------------------------------------------------------
.../GridCacheDatabaseSharedManager.java | 11 ++++-----
.../persistence/file/FilePageStoreManager.java | 12 +++++++---
.../cache/persistence/wal/FileDescriptor.java | 3 ++-
.../wal/FileWriteAheadLogManager.java | 23 +++++++++++--------
.../wal/FsyncModeFileWriteAheadLogManager.java | 24 ++++++++++++--------
.../file/IgnitePdsDiskErrorsRecoveringTest.java | 5 ++--
.../IgniteNodeStoppedDuringDisableWALTest.java | 5 ++--
.../persistence/db/wal/WalCompactionTest.java | 5 ++--
.../impl/v2/HadoopV2JobResourceManager.java | 3 ++-
.../submitter/command/StartCommand.java | 3 ++-
10 files changed, 57 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e7b86e8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index a86b9f5..f06fd05 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -217,9 +217,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** Checkpoint file name pattern. */
public static final Pattern CP_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-(.*)-(START|END)\\.bin");
- /** Checkpoint file temporary suffix. This is needed to safe writing checkpoint markers through temporary file and renaming. */
- public static final String FILE_TMP_SUFFIX = ".tmp";
-
/** Node started file suffix. */
public static final String NODE_STARTED_FILE_NAME_SUFFIX = "-node-started.bin";
@@ -498,13 +495,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
- * Cleanup checkpoint directory from all temporary files {@link #FILE_TMP_SUFFIX}.
+ * Cleanup checkpoint directory from all temporary files.
*/
@Override public void cleanupTempCheckpointDirectory() throws IgniteCheckedException {
try {
try (DirectoryStream<Path> files = Files.newDirectoryStream(
cpDir.toPath(),
- path -> path.endsWith(FILE_TMP_SUFFIX))
+ path -> path.endsWith(FilePageStoreManager.TMP_SUFFIX))
) {
for (Path path : files)
Files.delete(path);
@@ -886,7 +883,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
FileWALPointer p = (FileWALPointer)ptr;
String fileName = U.currentTimeMillis() + NODE_STARTED_FILE_NAME_SUFFIX;
- String tmpFileName = fileName + FILE_TMP_SUFFIX;
+ String tmpFileName = fileName + FilePageStoreManager.TMP_SUFFIX;
ByteBuffer buf = ByteBuffer.allocate(FileWALPointer.POINTER_SIZE);
buf.order(ByteOrder.nativeOrder());
@@ -2666,7 +2663,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
*/
public void writeCheckpointEntry(ByteBuffer entryBuf, CheckpointEntry cp, CheckpointEntryType type) throws StorageException {
String fileName = checkpointFileName(cp, type);
- String tmpFileName = fileName + FILE_TMP_SUFFIX;
+ String tmpFileName = fileName + FilePageStoreManager.TMP_SUFFIX;
try {
try (FileIO io = ioFactory.create(Paths.get(cpDir.getAbsolutePath(), skipSync ? fileName : tmpFileName).toFile(),
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e7b86e8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index aa09241..020f84e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -77,6 +77,12 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
/** File suffix. */
public static final String FILE_SUFFIX = ".bin";
+ /** Suffix for zip files */
+ public static final String ZIP_SUFFIX = ".zip";
+
+ /** Suffix for tmp files */
+ public static final String TMP_SUFFIX = ".tmp";
+
/** Partition file prefix. */
public static final String PART_FILE_PREFIX = "part-";
@@ -96,7 +102,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
public static final String CACHE_DATA_FILENAME = "cache_data.dat";
/** */
- public static final String CACHE_DATA_TMP_FILENAME = CACHE_DATA_FILENAME + ".tmp";
+ public static final String CACHE_DATA_TMP_FILENAME = CACHE_DATA_FILENAME + TMP_SUFFIX;
/** */
public static final String DFLT_STORE_DIR = "db";
@@ -341,7 +347,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
if (overwrite || !file.exists() || file.length() == 0) {
try {
- File tmp = new File(file.getParent(), file.getName() + ".tmp");
+ File tmp = new File(file.getParent(), file.getName() + TMP_SUFFIX);
tmp.createNewFile();
@@ -596,7 +602,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
Path cacheWorkDirPath = cacheWorkDir.toPath();
- Path tmp = cacheWorkDirPath.getParent().resolve(cacheWorkDir.getName() + ".tmp");
+ Path tmp = cacheWorkDirPath.getParent().resolve(cacheWorkDir.getName() + TMP_SUFFIX);
if (Files.exists(tmp) && Files.isDirectory(tmp) &&
Files.exists(tmp.resolve(IgniteCacheSnapshotManager.TEMP_FILES_COMPLETENESS_MARKER))) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e7b86e8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
index 39052cc..a73248a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache.persistence.wal;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.util.typedef.internal.SB;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -121,7 +122,7 @@ public class FileDescriptor implements Comparable<FileDescriptor>, AbstractWalRe
/** {@inheritDoc} */
@Override public boolean isCompressed() {
- return file.getName().endsWith(".zip");
+ return file.getName().endsWith(FilePageStoreManager.ZIP_SUFFIX);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e7b86e8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 61def1e..eb5ae95 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -92,6 +92,7 @@ import org.apache.ignite.internal.processors.cache.persistence.DataStorageMetric
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator.AbstractFileDescriptor;
@@ -534,7 +535,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
String segmentName = FileDescriptor.fileName(i);
File file = new File(walArchiveDir, segmentName);
- File fileZip = new File(walArchiveDir, segmentName + ".zip");
+ File fileZip = new File(walArchiveDir, segmentName + FilePageStoreManager.ZIP_SUFFIX);
if (file.exists())
res.add(file);
@@ -909,7 +910,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
private boolean hasIndex(long absIdx) {
String segmentName = FileDescriptor.fileName(absIdx);
- String zipSegmentName = FileDescriptor.fileName(absIdx) + ".zip";
+ String zipSegmentName = FileDescriptor.fileName(absIdx) + FilePageStoreManager.ZIP_SUFFIX;
boolean inArchive = new File(walArchiveDir, segmentName).exists() ||
new File(walArchiveDir, zipSegmentName).exists();
@@ -1496,7 +1497,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (log.isDebugEnabled())
log.debug("Creating new file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
- File tmp = new File(file.getParent(), file.getName() + ".tmp");
+ File tmp = new File(file.getParent(), file.getName() + FilePageStoreManager.TMP_SUFFIX);
formatFile(tmp);
@@ -1911,7 +1912,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
String name = FileDescriptor.fileName(absIdx);
- File dstTmpFile = new File(walArchiveDir, name + ".tmp");
+ File dstTmpFile = new File(walArchiveDir, name + FilePageStoreManager.TMP_SUFFIX);
File dstFile = new File(walArchiveDir, name);
@@ -2104,9 +2105,10 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (currReservedSegment == -1)
continue;
- File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + ".zip" + ".tmp");
+ File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment)
+ + FilePageStoreManager.ZIP_SUFFIX + FilePageStoreManager.TMP_SUFFIX);
- File zip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + ".zip");
+ File zip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + FilePageStoreManager.ZIP_SUFFIX);
File raw = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment));
if (!Files.exists(raw.toPath()))
@@ -2265,8 +2267,10 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (isCancelled())
break;
- File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".zip");
- File unzipTmp = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".tmp");
+ File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress)
+ + FilePageStoreManager.ZIP_SUFFIX);
+ File unzipTmp = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress)
+ + FilePageStoreManager.TMP_SUFFIX);
File unzip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress));
try (ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip)));
@@ -3061,7 +3065,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (!desc.file().exists()) {
FileDescriptor zipFile = new FileDescriptor(
- new File(walArchiveDir, FileDescriptor.fileName(desc.idx()) + ".zip"));
+ new File(walArchiveDir, FileDescriptor.fileName(desc.idx())
+ + FilePageStoreManager.ZIP_SUFFIX));
if (!zipFile.file.exists()) {
throw new FileNotFoundException("Both compressed and raw segment files are missing in archive " +
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e7b86e8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 91d3218..0f15d49 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -87,6 +87,7 @@ import org.apache.ignite.internal.processors.cache.persistence.DataStorageMetric
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
@@ -618,7 +619,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
String segmentName = FileDescriptor.fileName(i);
File file = new File(walArchiveDir, segmentName);
- File fileZip = new File(walArchiveDir, segmentName + ".zip");
+ File fileZip = new File(walArchiveDir, segmentName + FilePageStoreManager.ZIP_SUFFIX);
if (file.exists())
res.add(file);
@@ -815,7 +816,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
private boolean hasIndex(long absIdx) {
String segmentName = FileDescriptor.fileName(absIdx);
- String zipSegmentName = FileDescriptor.fileName(absIdx) + ".zip";
+ String zipSegmentName = FileDescriptor.fileName(absIdx) + FilePageStoreManager.ZIP_SUFFIX;
boolean inArchive = new File(walArchiveDir, segmentName).exists() ||
new File(walArchiveDir, zipSegmentName).exists();
@@ -1345,7 +1346,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
if (log.isDebugEnabled())
log.debug("Creating new file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
- File tmp = new File(file.getParent(), file.getName() + ".tmp");
+ File tmp = new File(file.getParent(), file.getName() + FilePageStoreManager.TMP_SUFFIX);
formatFile(tmp);
@@ -1740,7 +1741,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
String name = FileDescriptor.fileName(absIdx);
- File dstTmpFile = new File(walArchiveDir, name + ".tmp");
+ File dstTmpFile = new File(walArchiveDir, name + FilePageStoreManager.TMP_SUFFIX);
File dstFile = new File(walArchiveDir, name);
@@ -1934,9 +1935,11 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
if (currReservedSegment == -1)
continue;
- File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + ".zip" + ".tmp");
+ File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment)
+ + FilePageStoreManager.ZIP_SUFFIX + FilePageStoreManager.TMP_SUFFIX);
- File zip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + ".zip");
+ File zip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment)
+ + FilePageStoreManager.ZIP_SUFFIX);
File raw = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment));
if (!Files.exists(raw.toPath()))
@@ -2073,8 +2076,10 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
if (isCancelled())
break;
- File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".zip");
- File unzipTmp = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".tmp");
+ File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress)
+ + FilePageStoreManager.ZIP_SUFFIX);
+ File unzipTmp = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress)
+ + FilePageStoreManager.TMP_SUFFIX);
File unzip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress));
try (ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip)));
@@ -3146,7 +3151,8 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
if (!desc.file().exists()) {
FileDescriptor zipFile = new FileDescriptor(
- new File(walArchiveDir, FileDescriptor.fileName(desc.idx()) + ".zip"));
+ new File(walArchiveDir, FileDescriptor.fileName(desc.idx())
+ + FilePageStoreManager.ZIP_SUFFIX));
if (!zipFile.file.exists()) {
throw new FileNotFoundException("Both compressed and raw segment files are missing in archive " +
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e7b86e8/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
index 7efe29b..bd30696 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
@@ -44,6 +44,7 @@ import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabase
import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
@@ -158,7 +159,7 @@ public class IgnitePdsDiskErrorsRecoveringTest extends GridCommonAbstractTest {
*/
public void testRecoveringOnNodeStartMarkerWriteFail() throws Exception {
// Fail to write node start marker tmp file at the second checkpoint. Pass only initial checkpoint.
- ioFactory = new FilteringFileIOFactory("started.bin" + GridCacheDatabaseSharedManager.FILE_TMP_SUFFIX, new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 20));
+ ioFactory = new FilteringFileIOFactory("started.bin" + FilePageStoreManager.TMP_SUFFIX, new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 20));
IgniteEx grid = startGrid(0);
grid.cluster().active(true);
@@ -213,7 +214,7 @@ public class IgnitePdsDiskErrorsRecoveringTest extends GridCommonAbstractTest {
*/
public void testRecoveringOnCheckpointBeginFail() throws Exception {
// Fail to write checkpoint start marker tmp file at the second checkpoint. Pass only initial checkpoint.
- ioFactory = new FilteringFileIOFactory("START.bin" + GridCacheDatabaseSharedManager.FILE_TMP_SUFFIX, new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 20));
+ ioFactory = new FilteringFileIOFactory("START.bin" + FilePageStoreManager.TMP_SUFFIX, new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 20));
final IgniteEx grid = startGrid(0);
grid.cluster().active(true);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e7b86e8/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java
index 80198e8..a744ab1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java
@@ -37,6 +37,7 @@ import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -47,7 +48,7 @@ import org.junit.Assert;
import static java.nio.file.FileVisitResult.CONTINUE;
import static java.nio.file.Files.walkFileTree;
import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.CP_FILE_NAME_PATTERN;
-import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.FILE_TMP_SUFFIX;
+
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.META_STORAGE_NAME;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_PREFIX;
@@ -208,7 +209,7 @@ public class IgniteNodeStoppedDuringDisableWALTest extends GridCommonAbstractTes
boolean failed = false;
- if (name.endsWith(FILE_TMP_SUFFIX))
+ if (name.endsWith(FilePageStoreManager.TMP_SUFFIX))
failed = true;
if (CP_FILE_NAME_PATTERN.matcher(name).matches())
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e7b86e8/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
index f890d61..e617455 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
@@ -33,6 +33,7 @@ import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -163,7 +164,7 @@ public class WalCompactionTest extends GridCommonAbstractTest {
File walDir = new File(dbDir, "wal");
File archiveDir = new File(walDir, "archive");
File nodeArchiveDir = new File(archiveDir, nodeFolderName);
- File walSegment = new File(nodeArchiveDir, FileDescriptor.fileName(0) + ".zip");
+ File walSegment = new File(nodeArchiveDir, FileDescriptor.fileName(0) + FilePageStoreManager.ZIP_SUFFIX);
assertTrue(walSegment.exists());
assertTrue(walSegment.length() < WAL_SEGMENT_SIZE / 2); // Should be compressed at least in half.
@@ -358,7 +359,7 @@ public class WalCompactionTest extends GridCommonAbstractTest {
File walDir = new File(dbDir, "wal");
File archiveDir = new File(walDir, "archive");
File nodeArchiveDir = new File(archiveDir, nodeFolderName);
- File walSegment = new File(nodeArchiveDir, FileDescriptor.fileName(0) + ".zip");
+ File walSegment = new File(nodeArchiveDir, FileDescriptor.fileName(0) + FilePageStoreManager.ZIP_SUFFIX);
assertTrue(walSegment.exists());
assertTrue(walSegment.length() < WAL_SEGMENT_SIZE / 2); // Should be compressed at least in half.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e7b86e8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
index 52e394b..18dbf20 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.util.RunJar;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.hadoop.HadoopCommonUtils;
import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
import org.apache.ignite.internal.processors.hadoop.impl.fs.HadoopFileSystemsUtils;
@@ -234,7 +235,7 @@ class HadoopV2JobResourceManager {
if (archiveNameLC.endsWith(".jar"))
RunJar.unJar(archiveFile, dstPath);
- else if (archiveNameLC.endsWith(".zip"))
+ else if (archiveNameLC.endsWith(FilePageStoreManager.ZIP_SUFFIX))
FileUtil.unZip(archiveFile, dstPath);
else if (archiveNameLC.endsWith(".tar.gz") ||
archiveNameLC.endsWith(".tgz") ||
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e7b86e8/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StartCommand.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StartCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StartCommand.java
index 082b363..0202b8e 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StartCommand.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StartCommand.java
@@ -28,6 +28,7 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.commons.io.IOUtils;
import org.apache.ignite.Ignite;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.tensorflow.cluster.TensorFlowClusterGatewayManager;
import org.apache.ignite.tensorflow.cluster.TensorFlowJobArchive;
import picocli.CommandLine;
@@ -97,7 +98,7 @@ public class StartCommand extends AbstractCommand {
if (file.isDirectory())
return zipDirectory(file);
- else if (jobArchivePath.endsWith(".zip"))
+ else if (jobArchivePath.endsWith(FilePageStoreManager.ZIP_SUFFIX))
return zipArchive(file);
else
return zipFile(file);
[05/50] [abbrv] ignite git commit: IGNITE-9273 Log logical records to
WAL for LOCAL caches
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e9fd4e1/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
new file mode 100644
index 0000000..0b3ded4
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
@@ -0,0 +1,1911 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
+import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.internal.util.typedef.PAX;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.junit.Assert;
+
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
+/**
+ *
+ */
+public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
+ /** */
+ private static final String HAS_CACHE = "HAS_CACHE";
+
+ /** */
+ private static final int LARGE_ARR_SIZE = 1025;
+
+ /** */
+ private boolean fork;
+
+ /** */
+ private static final String CACHE_NAME = "partitioned";
+
+ /** */
+ private static final String RENAMED_CACHE_NAME = "partitioned0";
+
+ /** */
+ private static final String LOC_CACHE_NAME = "local";
+
+ /** */
+ private boolean renamed;
+
+ /** */
+ private int walSegmentSize;
+
+ /** Log only. */
+ private boolean logOnly;
+
+ /** {@inheritDoc} */
+ @Override protected boolean isMultiJvm() {
+ return fork;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration<Integer, IndexedObject> ccfg = renamed ?
+ new CacheConfiguration<>(RENAMED_CACHE_NAME) : new CacheConfiguration<>(CACHE_NAME);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+ ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+ ccfg.setNodeFilter(new RemoteNodeFilter());
+ ccfg.setIndexedTypes(Integer.class, IndexedObject.class);
+
+ CacheConfiguration<Integer, IndexedObject> locCcfg = new CacheConfiguration<>(LOC_CACHE_NAME);
+ locCcfg.setCacheMode(CacheMode.LOCAL);
+ locCcfg.setIndexedTypes(Integer.class, IndexedObject.class);
+
+ cfg.setCacheConfiguration(ccfg, locCcfg);
+
+ DataStorageConfiguration dbCfg = new DataStorageConfiguration();
+
+ dbCfg.setPageSize(4 * 1024);
+
+ DataRegionConfiguration memPlcCfg = new DataRegionConfiguration();
+
+ memPlcCfg.setName("dfltDataRegion");
+ memPlcCfg.setInitialSize(1024L * 1024 * 1024);
+ memPlcCfg.setMaxSize(1024L * 1024 * 1024);
+ memPlcCfg.setPersistenceEnabled(true);
+
+ dbCfg.setDefaultDataRegionConfiguration(memPlcCfg);
+
+ dbCfg.setWalRecordIteratorBufferSize(1024 * 1024);
+
+ dbCfg.setWalHistorySize(2);
+
+ if (logOnly)
+ dbCfg.setWalMode(WALMode.LOG_ONLY);
+
+ if (walSegmentSize != 0)
+ dbCfg.setWalSegmentSize(walSegmentSize);
+
+ cfg.setDataStorageConfiguration(dbCfg);
+
+ BinaryConfiguration binCfg = new BinaryConfiguration();
+
+ binCfg.setCompactFooter(false);
+
+ cfg.setBinaryConfiguration(binCfg);
+
+ if (!getTestIgniteInstanceName(0).equals(gridName))
+ cfg.setUserAttributes(F.asMap(HAS_CACHE, true));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+
+ renamed = false;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ logOnly = false;
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalBig() throws Exception {
+ IgniteEx ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+
+ Random rnd = new Random();
+
+ Map<Integer, IndexedObject> map = new HashMap<>();
+
+ for (int i = 0; i < 10_000; i++) {
+ if (i % 1000 == 0)
+ X.println(" >> " + i);
+
+ int k = rnd.nextInt(300_000);
+ IndexedObject v = new IndexedObject(rnd.nextInt(10_000));
+
+ cache.put(k, v);
+ map.put(k, v);
+ }
+
+ // Check.
+ for (Integer k : map.keySet())
+ assertEquals(map.get(k), cache.get(k));
+
+ stopGrid(1);
+
+ ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ cache = ignite.cache(CACHE_NAME);
+
+ // Check.
+ for (Integer k : map.keySet())
+ assertEquals(map.get(k), cache.get(k));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalBigObjectNodeCancel() throws Exception {
+ final int MAX_SIZE_POWER = 21;
+
+ IgniteEx ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < MAX_SIZE_POWER; ++i) {
+ int size = 1 << i;
+
+ cache.put("key_" + i, createTestData(size));
+ }
+
+ stopGrid(1, true);
+
+ ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ cache = ignite.cache(CACHE_NAME);
+
+ // Check.
+ for (int i = 0; i < MAX_SIZE_POWER; ++i) {
+ int size = 1 << i;
+
+ int[] data = createTestData(size);
+
+ int[] val = (int[])cache.get("key_" + i);
+
+ assertTrue("Invalid data. [key=key_" + i + ']', Arrays.equals(data, val));
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testSwitchClassLoader() throws Exception {
+ try {
+ final IgniteEx igniteEx = startGrid(1);
+
+ // CustomDiscoveryMessage will trigger service tasks
+ startGrid(2);
+
+ igniteEx.cluster().active(true);
+
+ IgniteCache<Integer, EnumVal> cache = igniteEx.cache(CACHE_NAME);
+
+ final ClassLoader oldCl = Thread.currentThread().getContextClassLoader();
+ final ClassLoader newCl = getExternalClassLoader();
+
+ Thread.currentThread().setContextClassLoader(newCl);
+
+ for (int i = 0; i < 10; i++)
+ cache.put(i, i % 2 == 0 ? EnumVal.VAL1 : EnumVal.VAL2);
+
+ for (int i = 0; i < 10; i++)
+ assert cache.containsKey(i);
+
+ // Invokes ClearTask with new class loader
+ cache.clear();
+
+ Thread.currentThread().setContextClassLoader(oldCl);
+
+ for (int i = 0; i < 10; i++)
+ cache.put(i, i % 2 == 0 ? EnumVal.VAL1 : EnumVal.VAL2);
+
+ for (int i = 0; i < 10; i++)
+ assert cache.containsKey(i);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalSimple() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+
+ info(" --> step1");
+
+ for (int i = 0; i < 10_000; i += 2)
+ cache.put(i, new IndexedObject(i));
+
+ info(" --> step2");
+
+ for (int i = 0; i < 10_000; i += 3)
+ cache.put(i, new IndexedObject(i * 2));
+
+ info(" --> step3");
+
+ for (int i = 0; i < 10_000; i += 7)
+ cache.put(i, new IndexedObject(i * 3));
+
+ info(" --> check1");
+
+ // Check.
+ for (int i = 0; i < 10_000; i++) {
+ IndexedObject o;
+
+ if (i % 7 == 0)
+ o = new IndexedObject(i * 3);
+ else if (i % 3 == 0)
+ o = new IndexedObject(i * 2);
+ else if (i % 2 == 0)
+ o = new IndexedObject(i);
+ else
+ o = null;
+
+ assertEquals(o, cache.get(i));
+ }
+
+ stopGrid(1);
+
+ ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ cache = ignite.cache(CACHE_NAME);
+
+ info(" --> check2");
+
+ // Check.
+ for (int i = 0; i < 10_000; i++) {
+ IndexedObject o;
+
+ if (i % 7 == 0)
+ o = new IndexedObject(i * 3);
+ else if (i % 3 == 0)
+ o = new IndexedObject(i * 2);
+ else if (i % 2 == 0)
+ o = new IndexedObject(i);
+ else
+ o = null;
+
+ assertEquals(o, cache.get(i));
+ }
+
+ info(" --> ok");
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testWalLargeValue() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < 10_000; i++) {
+ final byte[] data = new byte[i];
+
+ Arrays.fill(data, (byte)i);
+
+ cache.put(i, data);
+
+ if (i % 1000 == 0)
+ X.println(" ---> put: " + i);
+ }
+
+ stopGrid(1);
+
+ ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ cache = ignite.cache(CACHE_NAME);
+
+ info(" --> check2");
+
+ for (int i = 0; i < 10_000; i++) {
+ final byte[] data = new byte[i];
+
+ Arrays.fill(data, (byte)i);
+
+ final byte[] loaded = (byte[]) cache.get(i);
+
+ Assert.assertArrayEquals(data, loaded);
+
+ if (i % 1000 == 0)
+ X.println(" ---> get: " + i);
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalRolloverMultithreadedDefault() throws Exception {
+ logOnly = false;
+
+ checkWalRolloverMultithreaded();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalRolloverMultithreadedLogOnly() throws Exception {
+ logOnly = true;
+
+ checkWalRolloverMultithreaded();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testHugeCheckpointRecord() throws Exception {
+ try {
+ final IgniteEx ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ for (int i = 0; i < 50; i++) {
+ CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>("cache-" + i);
+
+ // We can get 'too many open files' with default number of partitions.
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 128));
+
+ IgniteCache<Object, Object> cache = ignite.getOrCreateCache(ccfg);
+
+ cache.put(i, i);
+ }
+
+ final long endTime = System.currentTimeMillis() + 30_000;
+
+ IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+ @Override public Void call() {
+ Random rnd = ThreadLocalRandom.current();
+
+ while (U.currentTimeMillis() < endTime) {
+ IgniteCache<Object, Object> cache = ignite.cache("cache-" + rnd.nextInt(50));
+
+ cache.put(rnd.nextInt(50_000), rnd.nextInt());
+ }
+
+ return null;
+ }
+ }, 16, "put-thread");
+
+ while (System.currentTimeMillis() < endTime) {
+ ignite.context().cache().context().database().wakeupForCheckpoint("test").get();
+
+ U.sleep(500);
+ }
+
+ fut.get();
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ private void checkWalRolloverMultithreaded() throws Exception {
+ walSegmentSize = 2 * 1024 * 1024;
+
+ final long endTime = System.currentTimeMillis() + 60 * 1000;
+
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ final IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+
+ GridTestUtils.runMultiThreaded(new Callable<Void>() {
+ @Override public Void call() {
+ Random rnd = ThreadLocalRandom.current();
+
+ while (U.currentTimeMillis() < endTime)
+ cache.put(rnd.nextInt(50_000), rnd.nextInt());
+
+ return null;
+ }
+ }, 16, "put-thread");
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testWalRenameDirSimple() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < 100; i++)
+ cache.put(i, new IndexedObject(i));
+
+ final Object consistentId = ignite.cluster().localNode().consistentId();
+
+ stopGrid(1);
+
+ final File cacheDir = cacheDir(CACHE_NAME, consistentId.toString());
+
+ renamed = cacheDir.renameTo(new File(cacheDir.getParent(), "cache-" + RENAMED_CACHE_NAME));
+
+ assert renamed;
+
+ ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ cache = ignite.cache(RENAMED_CACHE_NAME);
+
+ for (int i = 0; i < 100; i++)
+ assertEquals(new IndexedObject(i), cache.get(i));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @param consId Consistent ID.
+ * @return Cache dir.
+ * @throws IgniteCheckedException If fail.
+ */
+ private File cacheDir(final String cacheName, final String consId) throws IgniteCheckedException {
+ final String subfolderName
+ = PdsConsistentIdProcessor.genNewStyleSubfolderName(0, UUID.fromString(consId));
+
+ final File dbDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false);
+
+ assert dbDir.exists();
+
+ final File consIdDir = new File(dbDir.getAbsolutePath(), subfolderName);
+
+ assert consIdDir.exists();
+
+ final File cacheDir = new File(consIdDir.getAbsolutePath(), "cache-" + cacheName);
+
+ assert cacheDir.exists();
+
+ return cacheDir;
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRecoveryNoCheckpoint() throws Exception {
+ try {
+ IgniteEx ctrlGrid = startGrid(0);
+
+ fork = true;
+
+ IgniteEx cacheGrid = startGrid(1);
+
+ ctrlGrid.cluster().active(true);
+
+ ctrlGrid.compute(ctrlGrid.cluster().forRemotes()).run(new LoadRunnable(false));
+
+ info("Killing remote process...");
+
+ ((IgniteProcessProxy)cacheGrid).kill();
+
+ final IgniteEx g0 = ctrlGrid;
+
+ GridTestUtils.waitForCondition(new PA() {
+ /** {@inheritDoc} */
+ @Override public boolean apply() {
+ return g0.cluster().nodes().size() == 1;
+ }
+ }, getTestTimeout());
+
+ fork = false;
+
+ // Now start the grid and verify that updates were restored from WAL.
+ cacheGrid = startGrid(1);
+
+ IgniteCache<Object, Object> cache = cacheGrid.cache(CACHE_NAME);
+
+ for (int i = 0; i < 10_000; i++)
+ assertEquals(new IndexedObject(i), cache.get(i));
+
+ List<List<?>> res = cache.query(new SqlFieldsQuery("select count(iVal) from IndexedObject")).getAll();
+
+ assertEquals(1, res.size());
+ assertEquals(10_000L, res.get(0).get(0));
+
+ IgniteCache<Object, Object> locCache = cacheGrid.cache(LOC_CACHE_NAME);
+
+ for (int i = 0; i < 10_000; i++)
+ assertEquals(new IndexedObject(i), locCache.get(i));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRecoveryLargeNoCheckpoint() throws Exception {
+ try {
+ IgniteEx ctrlGrid = startGrid(0);
+
+ fork = true;
+
+ IgniteEx cacheGrid = startGrid(1);
+
+ ctrlGrid.cluster().active(true);
+
+ ctrlGrid.compute(ctrlGrid.cluster().forRemotes()).run(new LargeLoadRunnable(false));
+
+ info("Killing remote process...");
+
+ ((IgniteProcessProxy)cacheGrid).kill();
+
+ final IgniteEx g0 = ctrlGrid;
+
+ GridTestUtils.waitForCondition(new PA() {
+ /** {@inheritDoc} */
+ @Override public boolean apply() {
+ return g0.cluster().nodes().size() == 1;
+ }
+ }, getTestTimeout());
+
+ fork = false;
+
+ // Now start the grid and verify that updates were restored from WAL.
+ cacheGrid = startGrid(1);
+
+ IgniteCache<Object, Object> cache = cacheGrid.cache(CACHE_NAME);
+ IgniteCache<Object, Object> locCache = cacheGrid.cache(LOC_CACHE_NAME);
+
+ for (int i = 0; i < 1000; i++) {
+ final long[] data = new long[LARGE_ARR_SIZE];
+
+ Arrays.fill(data, i);
+
+ Assert.assertArrayEquals(data, (long[])cache.get(i));
+ Assert.assertArrayEquals(data, (long[])locCache.get(i));
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return TimeUnit.MINUTES.toMillis(20);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRandomCrash() throws Exception {
+ try {
+ IgniteEx ctrlGrid = startGrid(0);
+
+ fork = true;
+
+ IgniteEx cacheGrid = startGrid(1);
+
+ ctrlGrid.cluster().active(true);
+
+ IgniteCompute rmt = ctrlGrid.compute(ctrlGrid.cluster().forRemotes());
+
+ rmt.run(new LoadRunnable(false));
+
+ info(">>> Finished cache population.");
+
+ rmt.run(new AsyncLoadRunnable());
+
+ Thread.sleep(20_000);
+
+ info(">>> Killing remote process...");
+
+ ((IgniteProcessProxy)cacheGrid).kill();
+
+ startGrid(1);
+
+ Boolean res = rmt.call(new VerifyCallable());
+
+ assertTrue(res);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testLargeRandomCrash() throws Exception {
+ try {
+ IgniteEx ctrlGrid = startGrid(0);
+
+ fork = true;
+
+ IgniteEx cacheGrid = startGrid(1);
+
+ ctrlGrid.cluster().active(true);
+
+ IgniteCompute rmt = ctrlGrid.compute(ctrlGrid.cluster().forRemotes());
+
+ rmt.run(new LargeLoadRunnable(false));
+
+ info(">>> Finished cache population.");
+
+ rmt.run(new AsyncLargeLoadRunnable());
+
+ Thread.sleep(20_000);
+
+ info(">>> Killing remote process...");
+
+ ((IgniteProcessProxy)cacheGrid).kill();
+
+ startGrid(1);
+
+ Boolean res = rmt.call(new VerifyLargeCallable());
+
+ assertTrue(res);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ *
+ */
+ private static class RemoteNodeFilter implements IgnitePredicate<ClusterNode> {
+ /** {@inheritDoc} */
+ @Override public boolean apply(ClusterNode clusterNode) {
+ return clusterNode.attribute(HAS_CACHE) != null;
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDestroyCache() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ ignite.cluster().active(true);
+
+ IgniteCache<Object, Object> cache = ignite.getOrCreateCache("test");
+
+ cache.put(1, new IndexedObject(1));
+
+ ignite.destroyCache("test");
+
+ cache = ignite.getOrCreateCache("test");
+
+ // No entry available after cache destroy.
+ assertNull(cache.get(1));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testEvictPartition() throws Exception {
+ try {
+ Ignite ignite1 = startGrid("node1");
+
+ ignite1.cluster().active(true);
+
+ IgniteCache<Object, Object> cache1 = ignite1.cache(CACHE_NAME);
+
+ for (int i = 0; i < 100; i++)
+ cache1.put(i, new IndexedObject(i));
+
+ Ignite ignite2 = startGrid("node2");
+
+ IgniteCache<Object, Object> cache2 = ignite2.cache(CACHE_NAME);
+
+ for (int i = 0; i < 100; i++) {
+ assertEquals(new IndexedObject(i), cache1.get(i));
+ assertEquals(new IndexedObject(i), cache2.get(i));
+ }
+
+ ignite1.close();
+ ignite2.close();
+
+ ignite1 = startGrid("node1");
+ ignite2 = startGrid("node2");
+
+ ignite1.cluster().active(true);
+
+ cache1 = ignite1.cache(CACHE_NAME);
+ cache2 = ignite2.cache(CACHE_NAME);
+
+ for (int i = 0; i < 100; i++) {
+ assertEquals(new IndexedObject(i), cache1.get(i));
+ assertEquals(new IndexedObject(i), cache2.get(i));
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testMetastorage() throws Exception {
+ try {
+ int cnt = 5000;
+
+ IgniteEx ignite0 = (IgniteEx)startGrid("node1");
+ IgniteEx ignite1 = (IgniteEx)startGrid("node2");
+
+ ignite1.cluster().active(true);
+
+ GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
+ GridCacheSharedContext<Object, Object> sharedCtx1 = ignite1.context().cache().context();
+
+ MetaStorage storage0 = sharedCtx0.database().metaStorage();
+ MetaStorage storage1 = sharedCtx1.database().metaStorage();
+
+ assert storage0 != null;
+
+ for (int i = 0; i < cnt; i++) {
+ sharedCtx0.database().checkpointReadLock();
+
+ try {
+ storage0.putData(String.valueOf(i), new byte[]{(byte)(i % 256), 2, 3});
+ }
+ finally {
+ sharedCtx0.database().checkpointReadUnlock();
+ }
+
+ byte[] b1 = new byte[i + 3];
+ b1[0] = 1;
+ b1[1] = 2;
+ b1[2] = 3;
+
+ sharedCtx1.database().checkpointReadLock();
+
+ try {
+ storage1.putData(String.valueOf(i), b1);
+ }
+ finally {
+ sharedCtx1.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 0; i < cnt; i++) {
+ byte[] d1 = storage0.getData(String.valueOf(i));
+ assertEquals(3, d1.length);
+ assertEquals((byte)(i % 256), d1[0]);
+ assertEquals(2, d1[1]);
+ assertEquals(3, d1[2]);
+
+ byte[] d2 = storage1.getData(String.valueOf(i));
+ assertEquals(i + 3, d2.length);
+ assertEquals(1, d2[0]);
+ assertEquals(2, d2[1]);
+ assertEquals(3, d2[2]);
+ }
+
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testMetastorageLargeArray() throws Exception {
+ try {
+ int cnt = 5000;
+ int arraySize = 32_768;
+
+ IgniteEx ignite = (IgniteEx)startGrid("node1");
+
+ ignite.cluster().active(true);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = ignite.context().cache().context();
+
+ MetaStorage storage = sharedCtx.database().metaStorage();
+
+ for (int i = 0; i < cnt; i++) {
+ byte[] b1 = new byte[arraySize];
+ for (int k = 0; k < arraySize; k++) {
+ b1[k] = (byte) (k % 100);
+ }
+
+ sharedCtx.database().checkpointReadLock();
+
+ try {
+ storage.putData(String.valueOf(i), b1);
+ }
+ finally {
+ sharedCtx.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 0; i < cnt; i++) {
+ byte[] d2 = storage.getData(String.valueOf(i));
+ assertEquals(arraySize, d2.length);
+
+ for (int k = 0; k < arraySize; k++) {
+ assertEquals((byte) (k % 100), d2[k]);
+ }
+ }
+
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testMetastorageRemove() throws Exception {
+ try {
+ int cnt = 400;
+
+ IgniteEx ignite0 = (IgniteEx)startGrid("node1");
+
+ ignite0.cluster().active(true);
+
+ GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
+
+ MetaStorage storage = sharedCtx0.database().metaStorage();
+
+ assert storage != null;
+
+ for (int i = 0; i < cnt; i++) {
+ sharedCtx0.database().checkpointReadLock();
+
+ try {
+ storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
+ }
+ finally {
+ sharedCtx0.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 0; i < 10; i++) {
+ sharedCtx0.database().checkpointReadLock();
+
+ try {
+ storage.removeData(String.valueOf(i));
+ }
+ finally {
+ sharedCtx0.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 10; i < cnt; i++) {
+ byte[] d1 = storage.getData(String.valueOf(i));
+ assertEquals(3, d1.length);
+ assertEquals(1, d1[0]);
+ assertEquals(2, d1[1]);
+ assertEquals(3, d1[2]);
+ }
+
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testMetastorageUpdate() throws Exception {
+ try {
+ int cnt = 2000;
+
+ IgniteEx ignite0 = (IgniteEx)startGrid("node1");
+
+ ignite0.cluster().active(true);
+
+ GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
+
+ MetaStorage storage = sharedCtx0.database().metaStorage();
+
+ assert storage != null;
+
+ for (int i = 0; i < cnt; i++) {
+ sharedCtx0.database().checkpointReadLock();
+
+ try {
+ storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
+ }
+ finally {
+ sharedCtx0.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 0; i < cnt; i++) {
+ sharedCtx0.database().checkpointReadLock();
+
+ try {
+ storage.putData(String.valueOf(i), new byte[]{2, 2, 3, 4});
+ }
+ finally {
+ sharedCtx0.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 0; i < cnt; i++) {
+ byte[] d1 = storage.getData(String.valueOf(i));
+ assertEquals(4, d1.length);
+ assertEquals(2, d1[0]);
+ assertEquals(2, d1[1]);
+ assertEquals(3, d1[2]);
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testMetastorageWalRestore() throws Exception {
+ try {
+ int cnt = 2000;
+
+ IgniteEx ignite0 = startGrid(0);
+
+ ignite0.cluster().active(true);
+
+ GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
+
+ MetaStorage storage = sharedCtx0.database().metaStorage();
+
+ assert storage != null;
+
+ for (int i = 0; i < cnt; i++) {
+ sharedCtx0.database().checkpointReadLock();
+
+ try {
+ storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
+ }
+ finally {
+ sharedCtx0.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 0; i < cnt; i++) {
+ byte[] value = storage.getData(String.valueOf(i));
+ assert value != null;
+ assert value.length == 3;
+ }
+
+ stopGrid(0);
+
+ ignite0 = startGrid(0);
+
+ ignite0.cluster().active(true);
+
+ sharedCtx0 = ignite0.context().cache().context();
+
+ storage = sharedCtx0.database().metaStorage();
+
+ assert storage != null;
+
+ for (int i = 0; i < cnt; i++) {
+ byte[] value = storage.getData(String.valueOf(i));
+ assert value != null;
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testApplyDeltaRecords() throws Exception {
+ try {
+ IgniteEx ignite0 = (IgniteEx)startGrid("node0");
+
+ ignite0.cluster().active(true);
+
+ IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
+
+ for (int i = 0; i < 1000; i++)
+ cache0.put(i, new IndexedObject(i));
+
+ GridCacheSharedContext<Object, Object> sharedCtx = ignite0.context().cache().context();
+
+ GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)sharedCtx.database();
+
+ db.waitForCheckpoint("test");
+ db.enableCheckpoints(false).get();
+
+ // Log something to know where to start.
+ WALPointer ptr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
+
+ info("Replay marker: " + ptr);
+
+ for (int i = 1000; i < 5000; i++)
+ cache0.put(i, new IndexedObject(i));
+
+ info("Done puts...");
+
+ for (int i = 2_000; i < 3_000; i++)
+ cache0.remove(i);
+
+ info("Done removes...");
+
+ for (int i = 5000; i < 6000; i++)
+ cache0.put(i, new IndexedObject(i));
+
+ info("Done puts...");
+
+ Map<FullPageId, byte[]> rolledPages = new HashMap<>();
+
+ int pageSize = sharedCtx.database().pageSize();
+
+ ByteBuffer buf = ByteBuffer.allocateDirect(pageSize);
+
+ // Now check that deltas can be correctly applied.
+ try (WALIterator it = sharedCtx.wal().replay(ptr)) {
+ while (it.hasNext()) {
+ IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
+
+ WALRecord rec = tup.get2();
+
+ if (rec instanceof PageSnapshot) {
+ PageSnapshot page = (PageSnapshot)rec;
+
+ rolledPages.put(page.fullPageId(), page.pageData());
+ }
+ else if (rec instanceof PageDeltaRecord) {
+ PageDeltaRecord delta = (PageDeltaRecord)rec;
+
+ FullPageId fullId = new FullPageId(delta.pageId(), delta.groupId());
+
+ byte[] pageData = rolledPages.get(fullId);
+
+ if (pageData == null) {
+ pageData = new byte[pageSize];
+
+ rolledPages.put(fullId, pageData);
+ }
+
+ assertNotNull("Missing page snapshot [page=" + fullId + ", delta=" + delta + ']', pageData);
+
+ buf.order(ByteOrder.nativeOrder());
+
+ buf.position(0);
+ buf.put(pageData);
+ buf.position(0);
+
+ delta.applyDelta(sharedCtx.database().dataRegion(null).pageMemory(),
+ GridUnsafe.bufferAddress(buf));
+
+ buf.position(0);
+
+ buf.get(pageData);
+ }
+ }
+ }
+
+ info("Done apply...");
+
+ PageMemoryEx pageMem = (PageMemoryEx)db.dataRegion(null).pageMemory();
+
+ for (Map.Entry<FullPageId, byte[]> entry : rolledPages.entrySet()) {
+ FullPageId fullId = entry.getKey();
+
+ ignite0.context().cache().context().database().checkpointReadLock();
+
+ try {
+ long page = pageMem.acquirePage(fullId.groupId(), fullId.pageId(), true);
+
+ try {
+ long bufPtr = pageMem.writeLock(fullId.groupId(), fullId.pageId(), page, true);
+
+ try {
+ byte[] data = entry.getValue();
+
+ for (int i = 0; i < data.length; i++) {
+ if (fullId.pageId() == TrackingPageIO.VERSIONS.latest().trackingPageFor(fullId.pageId(), db.pageSize()))
+ continue; // Skip tracking pages.
+
+ assertEquals("page=" + fullId + ", pos=" + i, PageUtils.getByte(bufPtr, i), data[i]);
+ }
+ }
+ finally {
+ pageMem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, false, true);
+ }
+ }
+ finally {
+ pageMem.releasePage(fullId.groupId(), fullId.pageId(), page);
+ }
+ }
+ finally {
+ ignite0.context().cache().context().database().checkpointReadUnlock();
+ }
+ }
+
+ ignite0.close();
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * Test recovery from WAL on 3 nodes in case of transactional cache.
+ *
+ * @throws Exception If fail.
+ */
+ public void testRecoveryOnTransactionalAndPartitionedCache() throws Exception {
+ IgniteEx ignite = (IgniteEx) startGrids(3);
+ ignite.cluster().active(true);
+
+ try {
+ final String cacheName = "transactional";
+
+ CacheConfiguration<Object, Object> cacheConfiguration = new CacheConfiguration<>(cacheName)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setAffinity(new RendezvousAffinityFunction(false, 32))
+ .setCacheMode(CacheMode.PARTITIONED)
+ .setRebalanceMode(CacheRebalanceMode.SYNC)
+ .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+ .setBackups(2);
+
+ ignite.createCache(cacheConfiguration);
+
+ IgniteCache<Object, Object> cache = ignite.cache(cacheName);
+ Map<Object, Object> map = new HashMap<>();
+
+ final int transactions = 100;
+ final int operationsPerTransaction = 40;
+
+ Random random = new Random();
+
+ for (int t = 1; t <= transactions; t++) {
+ Transaction tx = ignite.transactions().txStart(
+ TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
+
+ Map<Object, Object> changesInTransaction = new HashMap<>();
+
+ for (int op = 0; op < operationsPerTransaction; op++) {
+ int key = random.nextInt(1000) + 1;
+
+ Object value = random.nextBoolean() ? randomString(random) + key : new BigObject(key);
+
+ changesInTransaction.put(key, value);
+
+ cache.put(key, value);
+ }
+
+ if (random.nextBoolean()) {
+ tx.commit();
+ map.putAll(changesInTransaction);
+ }
+ else {
+ tx.rollback();
+ }
+
+ if (t % 50 == 0)
+ log.info("Finished transaction " + t);
+ }
+
+ stopAllGrids();
+
+ ignite = (IgniteEx) startGrids(3);
+ ignite.cluster().active(true);
+
+ cache = ignite.cache(cacheName);
+
+ for (Object key : map.keySet()) {
+ Object expectedValue = map.get(key);
+ Object actualValue = cache.get(key);
+ Assert.assertEquals("Unexpected value for key " + key, expectedValue, actualValue);
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * Test that all DataRecord WAL records are within transaction boundaries - PREPARED and COMMITTED markers.
+ *
+ * @throws Exception If any fail.
+ */
+ public void testTxRecordsConsistency() throws Exception {
+ System.setProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS, "true");
+
+ IgniteEx ignite = (IgniteEx) startGrids(3);
+ ignite.cluster().active(true);
+
+ try {
+ final String cacheName = "transactional";
+
+ CacheConfiguration<Object, Object> cacheConfiguration = new CacheConfiguration<>(cacheName)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setAffinity(new RendezvousAffinityFunction(false, 32))
+ .setCacheMode(CacheMode.PARTITIONED)
+ .setRebalanceMode(CacheRebalanceMode.SYNC)
+ .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+ .setBackups(0);
+
+ ignite.createCache(cacheConfiguration);
+
+ IgniteCache<Object, Object> cache = ignite.cache(cacheName);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = ignite.context().cache().context();
+
+ GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)sharedCtx.database();
+
+ db.waitForCheckpoint("test");
+ db.enableCheckpoints(false).get();
+
+ // Log something to know where to start.
+ WALPointer startPtr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
+
+ final int transactions = 100;
+ final int operationsPerTransaction = 40;
+
+ Random random = new Random();
+
+ for (int t = 1; t <= transactions; t++) {
+ Transaction tx = ignite.transactions().txStart(
+ TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
+
+ for (int op = 0; op < operationsPerTransaction; op++) {
+ int key = random.nextInt(1000) + 1;
+
+ Object value = random.nextBoolean() ? randomString(random) + key : new BigObject(key);
+
+ cache.put(key, value);
+ }
+
+ if (random.nextBoolean()) {
+ tx.commit();
+ }
+ else {
+ tx.rollback();
+ }
+
+ if (t % 50 == 0)
+ log.info("Finished transaction " + t);
+ }
+
+ Set<GridCacheVersion> activeTransactions = new HashSet<>();
+
+ // Check that all DataRecords are within PREPARED and COMMITTED tx records.
+ try (WALIterator it = sharedCtx.wal().replay(startPtr)) {
+ while (it.hasNext()) {
+ IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
+
+ WALRecord rec = tup.get2();
+
+ if (rec instanceof TxRecord) {
+ TxRecord txRecord = (TxRecord) rec;
+ GridCacheVersion txId = txRecord.nearXidVersion();
+
+ switch (txRecord.state()) {
+ case PREPARED:
+ assert !activeTransactions.contains(txId) : "Transaction is already present " + txRecord;
+
+ activeTransactions.add(txId);
+
+ break;
+ case COMMITTED:
+ assert activeTransactions.contains(txId) : "No PREPARE marker for transaction " + txRecord;
+
+ activeTransactions.remove(txId);
+
+ break;
+ case ROLLED_BACK:
+ activeTransactions.remove(txId);
+ break;
+
+ default:
+ throw new IllegalStateException("Unknown Tx state of record " + txRecord);
+ }
+ } else if (rec instanceof DataRecord) {
+ DataRecord dataRecord = (DataRecord) rec;
+
+ for (DataEntry entry : dataRecord.writeEntries()) {
+ GridCacheVersion txId = entry.nearXidVersion();
+
+ assert activeTransactions.contains(txId) : "No transaction for entry " + entry;
+ }
+ }
+ }
+ }
+ }
+ finally {
+ System.clearProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS);
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * Generate random lowercase string for test purposes.
+ */
+ private String randomString(Random random) {
+ int len = random.nextInt(50) + 1;
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < len; i++)
+ sb.append(random.nextInt(26) + 'a');
+
+ return sb.toString();
+ }
+
+ /**
+ * BigObject for test purposes that don't fit in page size.
+ */
+ private static class BigObject {
+ private final int index;
+
+ private final byte[] payload = new byte[4096];
+
+ BigObject(int index) {
+ this.index = index;
+ // Create pseudo-random array.
+ for (int i = 0; i < payload.length; i++)
+ if (i % index == 0)
+ payload[i] = (byte) index;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ BigObject bigObject = (BigObject) o;
+ return index == bigObject.index &&
+ Arrays.equals(payload, bigObject.payload);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(index, payload);
+ }
+ }
+
+ /**
+ * @param size Size of data.
+ * @return Test data.
+ */
+ private int[] createTestData(int size) {
+ int[] data = new int[size];
+
+ for (int d = 0; d < size; ++d)
+ data[d] = d;
+
+ return data;
+ }
+
+ /**
+ *
+ */
+ private static class LoadRunnable implements IgniteRunnable {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** */
+ private boolean disableCheckpoints;
+
+ /**
+ * @param disableCheckpoints Disable checkpoints flag.
+ */
+ private LoadRunnable(boolean disableCheckpoints) {
+ this.disableCheckpoints = disableCheckpoints;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ ignite.log().info("Started load.");
+
+ if (disableCheckpoints) {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ try {
+ dbMgr.enableCheckpoints(false).get();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ try {
+ boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ return ignite.cache(CACHE_NAME) != null;
+ }
+ }, 10_000);
+
+ assertTrue(successfulWaiting);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+ IgniteCache<Object, Object> locCache = ignite.cache(LOC_CACHE_NAME);
+
+ for (int i = 0; i < 10_000; i++) {
+ cache.put(i, new IndexedObject(i));
+ locCache.put(i, new IndexedObject(i));
+ }
+
+ ignite.log().info("Finished load.");
+ }
+ }
+
+ /**
+ *
+ */
+ private static class AsyncLoadRunnable implements IgniteRunnable {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try {
+ boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ return ignite.cache(CACHE_NAME) != null;
+ }
+ }, 10_000);
+
+ assertTrue(successfulWaiting);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ ignite.log().info(">>>>>>> Started load.");
+
+ for (int i = 0; i < 4; i++) {
+ ignite.scheduler().callLocal(new Callable<Object>() {
+ @Override public Object call() {
+ IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+ IgniteCache<Object, Object> locCache = ignite.cache(LOC_CACHE_NAME);
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ int cnt = 0;
+
+ while (!Thread.currentThread().isInterrupted()) {
+ cache.put(rnd.nextInt(10_000), new IndexedObject(rnd.nextInt()));
+ locCache.put(rnd.nextInt(10_000), new IndexedObject(rnd.nextInt()));
+
+ cnt++;
+
+ if (cnt > 0 && cnt % 1_000 == 0)
+ ignite.log().info(">>>> Updated: " + cnt);
+ }
+
+ return null;
+ }
+ });
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private static class VerifyCallable implements IgniteCallable<Boolean> {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public Boolean call() throws Exception {
+ try {
+ boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ return ignite.cache(CACHE_NAME) != null;
+ }
+ }, 10_000);
+
+ assertTrue(successfulWaiting);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+ IgniteCache<Object, Object> locCache = ignite.cache(LOC_CACHE_NAME);
+
+ for (int i = 0; i < 10_000; i++) {
+ {
+ Object val = cache.get(i);
+
+ if (val == null) {
+ ignite.log().warning("Failed to find a value for PARTITIONED cache key: " + i);
+
+ return false;
+ }
+ }
+
+ {
+ Object val = locCache.get(i);
+
+ if (val == null) {
+ ignite.log().warning("Failed to find a value for LOCAL cache key: " + i);
+
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+ }
+
+ /**
+ *
+ */
+ private static class LargeLoadRunnable implements IgniteRunnable {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** */
+ private boolean disableCheckpoints;
+
+ /**
+ * @param disableCheckpoints Disable checkpoints flag.
+ */
+ private LargeLoadRunnable(boolean disableCheckpoints) {
+ this.disableCheckpoints = disableCheckpoints;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try {
+ boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ return ignite.cache(CACHE_NAME) != null;
+ }
+ }, 10_000);
+
+ assertTrue(successfulWaiting);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ ignite.log().info("Started load.");
+
+ if (disableCheckpoints) {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.enableCheckpoints(false);
+ }
+
+ IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+ IgniteCache<Object, Object> locCache = ignite.cache(LOC_CACHE_NAME);
+
+ for (int i = 0; i < 1000; i++) {
+ final long[] data = new long[LARGE_ARR_SIZE];
+
+ Arrays.fill(data, i);
+
+ cache.put(i, data);
+ locCache.put(i, data);
+ }
+
+ ignite.log().info("Finished load.");
+ }
+ }
+
+ /**
+ *
+ */
+ private static class AsyncLargeLoadRunnable implements IgniteRunnable {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try {
+ boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ return ignite.cache(CACHE_NAME) != null;
+ }
+ }, 10_000);
+
+ assertTrue(successfulWaiting);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ ignite.log().info(">>>>>>> Started load.");
+
+ for (int i = 0; i < 1; i++) {
+ ignite.scheduler().callLocal(new Callable<Object>() {
+ @Override public Object call() {
+ IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ int cnt = 0;
+
+ while (!Thread.currentThread().isInterrupted()) {
+ final long[] data = new long[LARGE_ARR_SIZE];
+
+ final int key = rnd.nextInt(1000);
+
+ Arrays.fill(data, key);
+
+ cache.put(key, data);
+
+ cnt++;
+
+ if (cnt > 0 && cnt % 1_000 == 0)
+ ignite.log().info(">>>> Updated: " + cnt);
+ }
+
+ return null;
+ }
+ });
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private static class VerifyLargeCallable implements IgniteCallable<Boolean> {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public Boolean call() throws Exception {
+ try {
+ boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ return ignite.cache(CACHE_NAME) != null;
+ }
+ }, 10_000);
+
+ assertTrue(successfulWaiting);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < 1000; i++) {
+ final long[] data = new long[LARGE_ARR_SIZE];
+
+ Arrays.fill(data, i);
+
+ final Object val = cache.get(i);
+
+ if (val == null) {
+ ignite.log().warning("Failed to find a value for key: " + i);
+
+ return false;
+ }
+
+ assertTrue(Arrays.equals(data, (long[])val));
+ }
+
+ return true;
+ }
+ }
+
+
+ /**
+ *
+ */
+ private static class IndexedObject {
+ /** */
+ @QuerySqlField(index = true)
+ private int iVal;
+
+ /**
+ * @param iVal Integer value.
+ */
+ private IndexedObject(int iVal) {
+ this.iVal = iVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof IndexedObject))
+ return false;
+
+ IndexedObject that = (IndexedObject)o;
+
+ return iVal == that.iVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return iVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IndexedObject.class, this);
+ }
+ }
+
+ /**
+ *
+ */
+ private enum EnumVal {
+ /** */
+ VAL1,
+
+ /** */
+ VAL2,
+
+ /** */
+ VAL3
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e9fd4e1/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryWithCompactionTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryWithCompactionTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryWithCompactionTest.java
new file mode 100644
index 0000000..d927676
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryWithCompactionTest.java
@@ -0,0 +1,33 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public class IgniteWalRecoveryWithCompactionTest extends IgniteWalRecoveryTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.getDataStorageConfiguration().setWalCompactionEnabled(true);
+
+ return cfg;
+ }
+}