You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2018/08/03 09:58:14 UTC
[01/50] [abbrv] ignite git commit: IGNITE-8797 Exception during
writeCheckpointEntry is passed to failure handler during checkpoint finish
Repository: ignite
Updated Branches:
refs/heads/ignite-8446 c2426df39 -> fa1a33ec6
IGNITE-8797 Exception during writeCheckpointEntry is passed to failure handler during checkpoint finish
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a61e1a90
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a61e1a90
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a61e1a90
Branch: refs/heads/ignite-8446
Commit: a61e1a90ccf65082bf49d481e5d88de5806ebaee
Parents: 774ed77
Author: Aleksey Plekhanov <pl...@gmail.com>
Authored: Mon Jul 30 22:43:37 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Mon Jul 30 22:43:37 2018 +0300
----------------------------------------------------------------------
.../GridCacheDatabaseSharedManager.java | 46 ++-------
.../IgniteClusterActivateDeactivateTest.java | 2 +
.../PageEvictionMultinodeMixedRegionsTest.java | 2 +
.../IgnitePdsCorruptedStoreTest.java | 99 +++++++++++++++++++-
...lientAffinityAssignmentWithBaselineTest.java | 4 +-
.../IgniteBaselineAbstractFullApiSelfTest.java | 2 +
...nitePdsCacheDestroyDuringCheckpointTest.java | 4 +-
.../db/file/IgnitePdsPageReplacementTest.java | 2 +
.../junits/common/GridCommonAbstractTest.java | 2 +
...ngBaselineCacheQueryNodeRestartSelfTest.java | 2 +
...eBaselineCacheQueryNodeRestartsSelfTest.java | 2 +
.../zk/internal/ZookeeperDiscoverySpiTest.java | 4 +-
12 files changed, 128 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 3c78e55..1203de8 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -3059,11 +3059,11 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
*
*/
private void doCheckpoint() {
+ Checkpoint chp = null;
+
try {
CheckpointMetricsTracker tracker = new CheckpointMetricsTracker();
- Checkpoint chp;
-
try {
chp = markCheckpointBegin(tracker);
}
@@ -3133,16 +3133,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
// Wait and check for errors.
- try {
- doneWriteFut.get();
- } catch (IgniteCheckedException e) {
- chp.progress.cpFinishFut.onDone(e);
-
- // In case of checkpoint writing error node should be invalidated and stopped.
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
-
- return;
- }
+ doneWriteFut.get();
// Must re-check shutdown flag here because threads may have skipped some pages.
// If so, we should not put finish checkpoint mark.
@@ -3175,33 +3166,14 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
snapshotMgr.afterCheckpointPageWritten();
- try {
- destroyedPartitionsCnt = destroyEvictedPartitions();
- }
- catch (IgniteCheckedException e) {
- chp.progress.cpFinishFut.onDone(e);
-
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
-
- return;
- }
+ destroyedPartitionsCnt = destroyEvictedPartitions();
// Must mark successful checkpoint only if there are no exceptions or interrupts.
success = true;
}
finally {
- if (success) {
- try {
- markCheckpointEnd(chp);
- }
- catch (IgniteCheckedException e) {
- chp.progress.cpFinishFut.onDone(e);
-
- cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
-
- return;
- }
- }
+ if (success)
+ markCheckpointEnd(chp);
}
tracker.onEnd();
@@ -3249,8 +3221,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
}
catch (IgniteCheckedException e) {
- // TODO-ignite-db how to handle exception?
- U.error(log, "Failed to create checkpoint.", e);
+ if (chp != null)
+ chp.progress.cpFinishFut.onDone(e);
+
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index cb2d13b..3b39bfb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -1128,6 +1128,8 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
* @throws Exception If failed.
*/
private void stateChangeFailover3(boolean activate) throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8220");
+
testReconnectSpi = true;
startNodesAndBlockStatusChange(4, 0, 0, !activate);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java
index 3da47e7..848e835 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java
@@ -53,6 +53,8 @@ public class PageEvictionMultinodeMixedRegionsTest extends PageEvictionMultinode
/** {@inheritDoc} */
@Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
cleanPersistenceDir();
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
index ff95f97..fc2a7d6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.cache.persistence;
import java.io.File;
import java.io.IOException;
+import java.nio.file.OpenOption;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
@@ -39,17 +40,24 @@ import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.wal.StorageException;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
-import org.apache.ignite.internal.processors.cache.persistence.file.PersistentStorageIOException;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiClosure;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID;
@@ -67,6 +75,9 @@ public class IgnitePdsCorruptedStoreTest extends GridCommonAbstractTest {
/** Failure handler. */
private DummyFailureHandler failureHnd;
+ /** Failing FileIO factory. */
+ private FailingFileIOFactory failingFileIOFactory;
+
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
cleanPersistenceDir();
@@ -91,12 +102,15 @@ public class IgnitePdsCorruptedStoreTest extends GridCommonAbstractTest {
cfg.setConsistentId(igniteInstanceName);
+ failingFileIOFactory = new FailingFileIOFactory();
+
DataStorageConfiguration memCfg = new DataStorageConfiguration()
.setDefaultDataRegionConfiguration(
new DataRegionConfiguration()
.setMaxSize(100 * 1024 * 1024)
.setPersistenceEnabled(true)
- );
+ )
+ .setFileIOFactory(failingFileIOFactory);
cfg.setDataStorageConfiguration(memCfg);
@@ -351,6 +365,48 @@ public class IgnitePdsCorruptedStoreTest extends GridCommonAbstractTest {
}
}
+
+ /**
+ * Test node invalidation due to checkpoint error.
+ */
+ public void testCheckpointFailure() throws Exception {
+ IgniteEx ignite = startGrid(0);
+
+ failingFileIOFactory.createClosure(new IgniteBiClosure<File, OpenOption[], FileIO>() {
+ @Override public FileIO apply(File file, OpenOption[] options) {
+ if (file.getName().indexOf("-END.bin") >= 0) {
+ FileIO delegate;
+
+ try {
+ delegate = failingFileIOFactory.delegateFactory().create(file, options);
+ }
+ catch (IOException ignore) {
+ return null;
+ }
+
+ return new FileIODecorator(delegate) {
+ @Override public void close() throws IOException {
+ throw new IOException("Checkpoint failed");
+ }
+ };
+ }
+
+ return null;
+ }
+ });
+
+ ignite.cluster().active(true);
+
+ try {
+ forceCheckpoint(ignite);
+ }
+ catch (Exception ignore) {
+ // No-op.
+ }
+
+ waitFailure(IOException.class);
+ }
+
/**
* @param expError Expected error.
*/
@@ -392,4 +448,43 @@ public class IgnitePdsCorruptedStoreTest extends GridCommonAbstractTest {
return true;
}
}
+
+ /**
+ * Create File I/O which can fail according to implemented closure.
+ */
+ private static class FailingFileIOFactory implements FileIOFactory {
+ /** Delegate factory. */
+ private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
+
+ /** Create FileIO closure. */
+ private volatile IgniteBiClosure<File, OpenOption[], FileIO> createClo;
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file) throws IOException {
+ return create(file, CREATE, READ, WRITE);
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file, OpenOption... openOption) throws IOException {
+ FileIO fileIO = null;
+ if (createClo != null)
+ fileIO = createClo.apply(file, openOption);
+
+ return fileIO != null ? fileIO : delegateFactory.create(file, openOption);
+ }
+
+ /**
+ * @param createClo FileIO create closure.
+ */
+ public void createClosure(IgniteBiClosure<File, OpenOption[], FileIO> createClo) {
+ this.createClo = createClo;
+ }
+
+ /**
+ *
+ */
+ public FileIOFactory delegateFactory() {
+ return delegateFactory;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
index 7071f65..7e9765c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
@@ -156,9 +156,9 @@ public class ClientAffinityAssignmentWithBaselineTest extends GridCommonAbstract
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
- cleanPersistenceDir();
-
stopAllGrids();
+
+ cleanPersistenceDir();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
index 67d13d6..5199fe9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
@@ -44,6 +44,8 @@ public abstract class IgniteBaselineAbstractFullApiSelfTest extends GridCacheAbs
@Override protected void afterTestsStopped() throws Exception {
super.afterTestsStopped();
+ stopAllGrids();
+
cleanPersistenceDir();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java
index bb20304..10fc386 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java
@@ -85,6 +85,8 @@ public class IgnitePdsCacheDestroyDuringCheckpointTest extends GridCommonAbstrac
/** {@inheritDoc} */
@Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
cleanPersistenceDir();
}
@@ -147,4 +149,4 @@ public class IgnitePdsCacheDestroyDuringCheckpointTest extends GridCommonAbstrac
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsPageReplacementTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsPageReplacementTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsPageReplacementTest.java
index f52b32e..432393e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsPageReplacementTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsPageReplacementTest.java
@@ -105,6 +105,8 @@ public class IgnitePdsPageReplacementTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
cleanPersistenceDir();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 33eae86..2c5091c 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -1712,6 +1712,8 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
*
*/
protected void cleanPersistenceDir() throws Exception {
+ assertTrue("Grids are not stopped", F.isEmpty(G.allGrids()));
+
U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "cp", false));
U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "marshaller", false));
http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
index b2a5758..b99745f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
@@ -65,6 +65,8 @@ public class IgniteChangingBaselineCacheQueryNodeRestartSelfTest extends IgniteC
/** {@inheritDoc} */
@Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
cleanPersistenceDir();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java
index d5a0408..0d631fc 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java
@@ -61,6 +61,8 @@ public class IgniteStableBaselineCacheQueryNodeRestartsSelfTest extends IgniteCa
/** {@inheritDoc} */
@Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
cleanPersistenceDir();
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index 6925a5e..cc80a72 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -479,9 +479,9 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
// checkZkNodesCleanup();
}
finally {
- reset();
-
stopAllGrids();
+
+ reset();
}
}
[41/50] [abbrv] ignite git commit: IGNITE-8714
CacheEntryEvent.getOldValue should be available
Posted by av...@apache.org.
IGNITE-8714 CacheEntryEvent.getOldValue should be available
Signed-off-by: Anton Vinogradov <av...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0b394056
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0b394056
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0b394056
Branch: refs/heads/ignite-8446
Commit: 0b394056b35e1874ef99f712462b5f16537751a8
Parents: 8811a14
Author: NSAmelchev <ns...@gmail.com>
Authored: Thu Aug 2 16:09:16 2018 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Thu Aug 2 16:09:16 2018 +0300
----------------------------------------------------------------------
.../datastructures/CacheDataStructuresManager.java | 5 +++--
.../continuous/CacheContinuousQueryManager.java | 4 ++--
.../processors/service/GridServiceProcessor.java | 5 +++--
.../cache/IgniteCacheEntryListenerAbstractTest.java | 16 ++++++++--------
...inuousQueryFactoryFilterRandomOperationTest.java | 6 +++---
.../CacheContinuousQueryRandomOperationsTest.java | 15 ++++++++-------
.../CacheContinuousQueryVariationsTest.java | 12 ++++++------
...ContinuousWithTransformerReplicatedSelfTest.java | 2 +-
.../GridCacheContinuousQueryAbstractSelfTest.java | 6 +++---
.../cpp/core-test/src/continuous_query_test.cpp | 6 +++---
.../Query/Continuous/ContinuousQueryAbstractTest.cs | 10 +++++-----
.../Impl/Cache/Event/CacheEntryRemoveEvent.cs | 4 ++--
.../Cache/Query/Continuous/ContinuousQueryUtils.cs | 2 +-
13 files changed, 48 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index ccfdc15..7d45c81 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -29,6 +29,7 @@ import java.util.Iterator;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -70,8 +71,8 @@ import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-import java.util.concurrent.ConcurrentHashMap;
+import static javax.cache.event.EventType.REMOVED;
import static org.apache.ignite.internal.GridClosureCallMode.BROADCAST;
/**
@@ -294,7 +295,7 @@ public class CacheDataStructuresManager extends GridCacheManagerAdapter {
for (final GridCacheQueueProxy queue : queuesMap.values()) {
if (queue.name().equals(key.queueName())) {
- if (hdr == null) {
+ if (e.getEventType() == REMOVED) {
GridCacheQueueHeader oldHdr = (GridCacheQueueHeader)e.getOldValue();
assert oldHdr != null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index c7635e0..4005dd8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -380,7 +380,7 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
cctx.cacheId(),
evtType,
key,
- newVal,
+ evtType == REMOVED && lsnr.oldValueRequired() ? oldVal : newVal,
lsnr.oldValueRequired() ? oldVal : null,
lsnr.keepBinary(),
partId,
@@ -442,7 +442,7 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
cctx.cacheId(),
EXPIRED,
key,
- null,
+ lsnr.oldValueRequired() ? oldVal : null,
lsnr.oldValueRequired() ? oldVal : null,
lsnr.keepBinary(),
e.partition(),
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 0f29791..949c0c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -111,6 +111,7 @@ import org.apache.ignite.thread.OomExceptionHandler;
import org.apache.ignite.transactions.Transaction;
import org.jetbrains.annotations.Nullable;
+import static javax.cache.event.EventType.REMOVED;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_SERVICES_COMPATIBILITY_MODE;
import static org.apache.ignite.IgniteSystemProperties.getString;
import static org.apache.ignite.configuration.DeploymentMode.ISOLATED;
@@ -1633,7 +1634,7 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
throw ex;
}
- if (dep != null) {
+ if (e.getEventType() != REMOVED) {
svcName.set(dep.configuration().getName());
// Ignore other utility cache events.
@@ -1980,7 +1981,7 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
throw ex;
}
- if (assigns != null) {
+ if (e.getEventType() != REMOVED) {
svcName.set(assigns.name());
Throwable t = null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java
index c6e3552..74b4253 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryListenerAbstractTest.java
@@ -314,7 +314,7 @@ public abstract class IgniteCacheEntryListenerAbstractTest extends IgniteCacheAb
syncEvent(key, null, cache, 1);
- checkEvent(evts.iterator(), key, REMOVED, null, 2);
+ checkEvent(evts.iterator(), key, REMOVED, 2, 2);
log.info("Check synchronous expire event [key=" + key + ']');
@@ -335,7 +335,7 @@ public abstract class IgniteCacheEntryListenerAbstractTest extends IgniteCacheAb
assertEquals(1, evts.size());
}
- checkEvent(evts.iterator(), key, EXPIRED, null, 3);
+ checkEvent(evts.iterator(), key, EXPIRED, 3, 3);
assertEquals(0, evts.size());
}
@@ -764,7 +764,7 @@ public abstract class IgniteCacheEntryListenerAbstractTest extends IgniteCacheAb
switch (evt.getEventType()) {
case REMOVED:
- assertNull(evt.getValue());
+ assertEquals(evt.getOldValue(), evt.getValue());
assertEquals(vals.get(evt.getKey()), evt.getOldValue());
@@ -797,7 +797,7 @@ public abstract class IgniteCacheEntryListenerAbstractTest extends IgniteCacheAb
break;
case EXPIRED:
- assertNull(evt.getValue());
+ assertEquals(evt.getOldValue(), evt.getValue());
assertEquals(value(-1), evt.getOldValue());
@@ -934,13 +934,13 @@ public abstract class IgniteCacheEntryListenerAbstractTest extends IgniteCacheAb
}
if (rmv)
- checkEvent(iter, key, REMOVED, null, oldVal ? UPDATES : null);
+ checkEvent(iter, key, REMOVED, oldVal ? UPDATES : null, oldVal ? UPDATES : null);
if (create)
checkEvent(iter, key, CREATED, 10, null);
if (expire)
- checkEvent(iter, key, EXPIRED, null, oldVal ? 10 : null);
+ checkEvent(iter, key, EXPIRED, oldVal ? 10 : null, oldVal ? 10 : null);
if (create)
checkEvent(iter, key, CREATED, 1, null);
@@ -949,13 +949,13 @@ public abstract class IgniteCacheEntryListenerAbstractTest extends IgniteCacheAb
checkEvent(iter, key, UPDATED, 2, oldVal ? 1 : null);
if (rmv)
- checkEvent(iter, key, REMOVED, null, oldVal ? 2 : null);
+ checkEvent(iter, key, REMOVED, oldVal ? 2 : null, oldVal ? 2 : null);
if (create)
checkEvent(iter, key, CREATED, 20, null);
if (expire)
- checkEvent(iter, key, EXPIRED, null, oldVal ? 20 : null);
+ checkEvent(iter, key, EXPIRED, oldVal ? 20 : null, oldVal ? 20 : null);
assertEquals(0, evts.size());
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFactoryFilterRandomOperationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFactoryFilterRandomOperationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFactoryFilterRandomOperationTest.java
index 69799ef..993ef79 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFactoryFilterRandomOperationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFactoryFilterRandomOperationTest.java
@@ -319,7 +319,7 @@ public class CacheContinuousQueryFactoryFilterRandomOperationTest extends CacheC
updatePartitionCounter(cache, key, partCntr);
- waitAndCheckEvent(evtsQueues, partCntr, affinity(cache), key, null, oldVal);
+ waitAndCheckEvent(evtsQueues, partCntr, affinity(cache), key, oldVal, oldVal);
expData.remove(key);
@@ -334,7 +334,7 @@ public class CacheContinuousQueryFactoryFilterRandomOperationTest extends CacheC
updatePartitionCounter(cache, key, partCntr);
- waitAndCheckEvent(evtsQueues, partCntr, affinity(cache), key, null, oldVal);
+ waitAndCheckEvent(evtsQueues, partCntr, affinity(cache), key, oldVal, oldVal);
expData.remove(key);
@@ -364,7 +364,7 @@ public class CacheContinuousQueryFactoryFilterRandomOperationTest extends CacheC
updatePartitionCounter(cache, key, partCntr);
- waitAndCheckEvent(evtsQueues, partCntr, affinity(cache), key, null, oldVal);
+ waitAndCheckEvent(evtsQueues, partCntr, affinity(cache), key, oldVal, oldVal);
expData.remove(key);
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryRandomOperationsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryRandomOperationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryRandomOperationsTest.java
index 1c6bd8c..6d15df2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryRandomOperationsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryRandomOperationsTest.java
@@ -81,6 +81,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static javax.cache.event.EventType.CREATED;
import static javax.cache.event.EventType.REMOVED;
+import static javax.cache.event.EventType.UPDATED;
import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
@@ -554,12 +555,12 @@ public class CacheContinuousQueryRandomOperationsTest extends GridCommonAbstract
}, 5_000);
checkSingleEvent(evts.get(0), CREATED, new QueryTestValue(1), null);
- checkSingleEvent(evts.get(1), REMOVED, null, new QueryTestValue(1));
+ checkSingleEvent(evts.get(1), REMOVED, new QueryTestValue(1), new QueryTestValue(1));
checkSingleEvent(evts.get(2), CREATED, new QueryTestValue(2), null);
- checkSingleEvent(evts.get(3), REMOVED, null, new QueryTestValue(2));
+ checkSingleEvent(evts.get(3), REMOVED, new QueryTestValue(2), new QueryTestValue(2));
checkSingleEvent(evts.get(4), CREATED, new QueryTestValue(3), null);
checkSingleEvent(evts.get(5), EventType.UPDATED, new QueryTestValue(4), new QueryTestValue(3));
- checkSingleEvent(evts.get(6), REMOVED, null, new QueryTestValue(4));
+ checkSingleEvent(evts.get(6), REMOVED, new QueryTestValue(4), new QueryTestValue(4));
checkSingleEvent(evts.get(7), CREATED, new QueryTestValue(5), null);
checkSingleEvent(evts.get(8), EventType.UPDATED, new QueryTestValue(6), new QueryTestValue(5));
@@ -699,7 +700,7 @@ public class CacheContinuousQueryRandomOperationsTest extends GridCommonAbstract
if (e.getKey().equals(keyVal)) {
checkSingleEvent(e,
evtType,
- evtType == CREATED ? new QueryTestValue(key) : null,
+ evtType != UPDATED ? new QueryTestValue(key) : null,
evtType == REMOVED ? new QueryTestValue(key) : null);
keyVal = null;
@@ -1032,7 +1033,7 @@ public class CacheContinuousQueryRandomOperationsTest extends GridCommonAbstract
updatePartitionCounter(cache, key, partCntr, expEvtCntrs);
- waitAndCheckEvent(evtsQueues, partCntr, expEvtCntrs, affinity(cache), key, null, oldVal);
+ waitAndCheckEvent(evtsQueues, partCntr, expEvtCntrs, affinity(cache), key, oldVal, oldVal);
expData.remove(key);
@@ -1047,7 +1048,7 @@ public class CacheContinuousQueryRandomOperationsTest extends GridCommonAbstract
updatePartitionCounter(cache, key, partCntr, expEvtCntrs);
- waitAndCheckEvent(evtsQueues, partCntr, expEvtCntrs, affinity(cache), key, null, oldVal);
+ waitAndCheckEvent(evtsQueues, partCntr, expEvtCntrs, affinity(cache), key, oldVal, oldVal);
expData.remove(key);
@@ -1077,7 +1078,7 @@ public class CacheContinuousQueryRandomOperationsTest extends GridCommonAbstract
updatePartitionCounter(cache, key, partCntr, expEvtCntrs);
- waitAndCheckEvent(evtsQueues, partCntr, expEvtCntrs, affinity(cache), key, null, oldVal);
+ waitAndCheckEvent(evtsQueues, partCntr, expEvtCntrs, affinity(cache), key, oldVal, oldVal);
expData.remove(key);
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryVariationsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryVariationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryVariationsTest.java
index ab086bc..fc86a35 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryVariationsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryVariationsTest.java
@@ -362,7 +362,7 @@ public class CacheContinuousQueryVariationsTest extends IgniteCacheConfigVariati
if (tx != null)
tx.commit();
- waitAndCheckEvent(evtsQueues, key, null, oldVal, keepBinary, withFilter);
+ waitAndCheckEvent(evtsQueues, key, oldVal, oldVal, keepBinary, withFilter);
expData.remove(key);
@@ -375,7 +375,7 @@ public class CacheContinuousQueryVariationsTest extends IgniteCacheConfigVariati
if (tx != null)
tx.commit();
- waitAndCheckEvent(evtsQueues, key, null, oldVal, keepBinary, withFilter);
+ waitAndCheckEvent(evtsQueues, key, oldVal, oldVal, keepBinary, withFilter);
expData.remove(key);
@@ -401,7 +401,7 @@ public class CacheContinuousQueryVariationsTest extends IgniteCacheConfigVariati
if (tx != null)
tx.commit();
- waitAndCheckEvent(evtsQueues, key, null, oldVal, keepBinary, withFilter);
+ waitAndCheckEvent(evtsQueues, key, oldVal, oldVal, keepBinary, withFilter);
expData.remove(key);
@@ -701,12 +701,12 @@ public class CacheContinuousQueryVariationsTest extends IgniteCacheConfigVariati
}, 5_000);
checkEvent(evts.get(0), CREATED, value(1), null);
- checkEvent(evts.get(1), REMOVED, null, value(1));
+ checkEvent(evts.get(1), REMOVED, value(1), value(1));
checkEvent(evts.get(2), CREATED, value(2), null);
- checkEvent(evts.get(3), REMOVED, null, value(2));
+ checkEvent(evts.get(3), REMOVED, value(2), value(2));
checkEvent(evts.get(4), CREATED, value(3), null);
checkEvent(evts.get(5), EventType.UPDATED, value(4), value(3));
- checkEvent(evts.get(6), REMOVED, null, value(4));
+ checkEvent(evts.get(6), REMOVED, value(4), value(4));
checkEvent(evts.get(7), CREATED, value(5), null);
checkEvent(evts.get(8), EventType.UPDATED, value(6), value(5));
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousWithTransformerReplicatedSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousWithTransformerReplicatedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousWithTransformerReplicatedSelfTest.java
index 7b53fb2..7aa91a2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousWithTransformerReplicatedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousWithTransformerReplicatedSelfTest.java
@@ -319,7 +319,7 @@ public class CacheContinuousWithTransformerReplicatedSelfTest extends GridCommon
qry.setRemoteTransformerFactory(FactoryBuilder.factoryOf(
new IgniteClosure<CacheEntryEvent<? extends Integer, ? extends Employee>, Integer>() {
@Override public Integer apply(CacheEntryEvent<? extends Integer, ? extends Employee> evt) {
- assertNull(evt.getValue());
+ assertNotNull(evt.getValue());
assertNotNull(evt.getOldValue());
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/GridCacheContinuousQueryAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/GridCacheContinuousQueryAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/GridCacheContinuousQueryAbstractSelfTest.java
index 61acf14..d09aea3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/GridCacheContinuousQueryAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/GridCacheContinuousQueryAbstractSelfTest.java
@@ -338,7 +338,7 @@ public abstract class GridCacheContinuousQueryAbstractSelfTest extends GridCommo
assertNotNull(vals);
assertEquals(2, vals.size());
assertEquals(2, (int)vals.get(0));
- assertNull(vals.get(1));
+ assertEquals(2, (int)vals.get(1));
vals = map.get(3);
@@ -564,7 +564,7 @@ public abstract class GridCacheContinuousQueryAbstractSelfTest extends GridCommo
assertNotNull(vals);
assertEquals(2, vals.size());
assertEquals(3, (int)vals.get(0));
- assertNull(vals.get(1));
+ assertEquals(3, (int)vals.get(1));
vals = map.get(4);
@@ -1136,7 +1136,7 @@ public abstract class GridCacheContinuousQueryAbstractSelfTest extends GridCommo
@Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts) {
for (CacheEntryEvent<?, ?> e : evts) {
if (e.getEventType() == EventType.EXPIRED) {
- assertNull(e.getValue());
+ assertEquals(e.getOldValue(), e.getValue());
map.put(e.getKey(), e.getOldValue());
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/platforms/cpp/core-test/src/continuous_query_test.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/core-test/src/continuous_query_test.cpp b/modules/platforms/cpp/core-test/src/continuous_query_test.cpp
index b869eb6..a5136d2 100644
--- a/modules/platforms/cpp/core-test/src/continuous_query_test.cpp
+++ b/modules/platforms/cpp/core-test/src/continuous_query_test.cpp
@@ -366,7 +366,7 @@ void CheckEvents(Cache<int, TestEntry>& cache, Listener<int, TestEntry>& lsnr)
lsnr.CheckNextEvent(2, boost::none, TestEntry(20));
cache.Remove(1);
- lsnr.CheckNextEvent(1, TestEntry(20), boost::none);
+ lsnr.CheckNextEvent(1, TestEntry(20), TestEntry(20));
}
IGNITE_EXPORTED_CALL void IgniteModuleInit0(ignite::IgniteBindingContext& context)
@@ -712,7 +712,7 @@ BOOST_AUTO_TEST_CASE(TestFilterSingleNode)
lsnr.CheckNextEvent(142, boost::none, TestEntry(1420));
lsnr.CheckNextEvent(142, TestEntry(1420), TestEntry(1421));
- lsnr.CheckNextEvent(142, TestEntry(1421), boost::none);
+ lsnr.CheckNextEvent(142, TestEntry(1421), TestEntry(1421));
lsnr.CheckNextEvent(149, boost::none, TestEntry(1490));
}
@@ -762,7 +762,7 @@ BOOST_AUTO_TEST_CASE(TestFilterMultipleNodes)
lsnr.CheckNextEvent(142, boost::none, TestEntry(1420));
lsnr.CheckNextEvent(142, TestEntry(1420), TestEntry(1421));
- lsnr.CheckNextEvent(142, TestEntry(1421), boost::none);
+ lsnr.CheckNextEvent(142, TestEntry(1421), TestEntry(1421));
lsnr.CheckNextEvent(149, boost::none, TestEntry(1490));
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Continuous/ContinuousQueryAbstractTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Continuous/ContinuousQueryAbstractTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Continuous/ContinuousQueryAbstractTest.cs
index 2df2f35..7326a82 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Continuous/ContinuousQueryAbstractTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Continuous/ContinuousQueryAbstractTest.cs
@@ -217,7 +217,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Continuous
CheckCallbackSingle(key1, Entry(key1), Entry(key1 + 1), CacheEntryEventType.Updated);
cache1.Remove(key1);
- CheckCallbackSingle(key1, Entry(key1 + 1), null, CacheEntryEventType.Removed);
+ CheckCallbackSingle(key1, Entry(key1 + 1), Entry(key1 + 1), CacheEntryEventType.Removed);
// Put from remote node.
cache2.GetAndPut(key2, Entry(key2));
@@ -239,7 +239,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Continuous
if (loc)
CheckNoCallback(100);
else
- CheckCallbackSingle(key2, Entry(key2 + 1), null, CacheEntryEventType.Removed);
+ CheckCallbackSingle(key2, Entry(key2 + 1), Entry(key2 + 1), CacheEntryEventType.Removed);
}
cache1.Put(key1, Entry(key1));
@@ -652,10 +652,10 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Continuous
Assert.IsTrue(CB_EVTS.TryTake(out cbEvt, 500));
cbEntry = cbEvt.entries.Single();
Assert.IsTrue(cbEntry.HasOldValue);
- Assert.IsFalse(cbEntry.HasValue);
+ Assert.IsTrue(cbEntry.HasValue);
Assert.AreEqual(key, cbEntry.Key);
Assert.AreEqual(2, cbEntry.OldValue);
- Assert.AreEqual(null, cbEntry.Value);
+ Assert.AreEqual(2, cbEntry.Value);
}
}
@@ -948,7 +948,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Continuous
if (!e.HasOldValue)
return new CacheEntryCreateEvent<object, object>(e.Key, e.Value);
- if (!e.HasValue)
+ if (e.Value.Equals(e.OldValue))
return new CacheEntryRemoveEvent<object, object>(e.Key, e.OldValue);
return new CacheEntryUpdateEvent<object, object>(e.Key, e.OldValue, e.Value);
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Event/CacheEntryRemoveEvent.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Event/CacheEntryRemoveEvent.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Event/CacheEntryRemoveEvent.cs
index cce3d65..eee1626 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Event/CacheEntryRemoveEvent.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Event/CacheEntryRemoveEvent.cs
@@ -50,7 +50,7 @@ namespace Apache.Ignite.Core.Impl.Cache.Event
/** <inheritdoc /> */
public TV Value
{
- get { return default(TV); }
+ get { return _oldVal; }
}
/** <inheritdoc /> */
@@ -62,7 +62,7 @@ namespace Apache.Ignite.Core.Impl.Cache.Event
/** <inheritdoc /> */
public bool HasValue
{
- get { return false; }
+ get { return true; }
}
/** <inheritdoc /> */
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b394056/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryUtils.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryUtils.cs
index 729b251..71b0580 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryUtils.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryUtils.cs
@@ -87,7 +87,7 @@ namespace Apache.Ignite.Core.Impl.Cache.Query.Continuous
if (!hasOldVal)
return new CacheEntryCreateEvent<TK, TV>(key, val);
- if (!hasVal)
+ if (val.Equals(oldVal))
return new CacheEntryRemoveEvent<TK, TV>(key, oldVal);
return new CacheEntryUpdateEvent<TK, TV>(key, oldVal, val);
[43/50] [abbrv] ignite git commit: Reference from github readme page
to TeamCity tests was updated.
Posted by av...@apache.org.
Reference from github readme page to TeamCity tests was updated.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/10b59144
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/10b59144
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/10b59144
Branch: refs/heads/ignite-8446
Commit: 10b591449eac443322cba60d7b84eb151cadb79b
Parents: 0e9bb1c
Author: Dmitriy Pavlov <dp...@apache.org>
Authored: Thu Aug 2 18:03:54 2018 +0300
Committer: Dmitriy Pavlov <dp...@apache.org>
Committed: Thu Aug 2 18:04:21 2018 +0300
----------------------------------------------------------------------
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/10b59144/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index c99ddbe..b4a9b23 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
<a href="https://ignite.apache.org/"><img src="https://ignite.apache.org/images/logo3.png" hspace="20"/></a>
-<a href="https://ci.ignite.apache.org/viewType.html?buildTypeId=Ignite20Tests_RunAll&branch_Ignite20Tests=%3Cdefault%3E"><img src="http://ci.ignite.apache.org/app/rest/builds/buildType:(id:Ignite20Tests_RunAll)/statusIcon.svg"/></a>
+<a href="https://ci.ignite.apache.org/viewType.html?buildTypeId=IgniteTests24Java8_RunAll&branch_IgniteTests24Java8=%3Cdefault%3E"><img src="https://ci.ignite.apache.org/app/rest/builds/buildType:(id:IgniteTests24Java8_RunAll)/statusIcon.svg"/></a>
## Getting Started
[20/50] [abbrv] ignite git commit: IGNITE-9114: SQL: use query time
in retry timeout calculation. This closes #4460.
Posted by av...@apache.org.
IGNITE-9114: SQL: use query time in retry timeout calculation. This closes #4460.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f0411759
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f0411759
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f0411759
Branch: refs/heads/ignite-8446
Commit: f0411759b884000dd30e137d0ba70c733651f914
Parents: bcda7a1
Author: devozerov <pp...@gmail.com>
Authored: Tue Jul 31 17:15:49 2018 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Tue Jul 31 17:15:49 2018 +0300
----------------------------------------------------------------------
.../query/h2/twostep/GridReduceQueryExecutor.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f0411759/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index d778fcc..39b2bbc 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -114,7 +114,7 @@ import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySpl
*/
public class GridReduceQueryExecutor {
/** Fail query after 10 seconds of unsuccessful attempts to reserve partitions. */
- public static final long DFLT_RETRY_TIMEOUT = 10_000L;
+ public static final long DFLT_RETRY_TIMEOUT = 30_000L;
/** */
private static final String MERGE_INDEX_UNSORTED = "merge_scan";
@@ -568,7 +568,7 @@ public class GridReduceQueryExecutor {
final boolean isReplicatedOnly = qry.isReplicatedOnly();
- long retryTimeout = retryTimeout();
+ long retryTimeout = retryTimeout(timeoutMillis);
final long startTime = U.currentTimeMillis();
@@ -1756,9 +1756,13 @@ public class GridReduceQueryExecutor {
}
/**
+ * @param qryTimeout Query timeout.
* @return Query retry timeout.
*/
- private static long retryTimeout() {
+ private static long retryTimeout(long qryTimeout) {
+ if (qryTimeout > 0)
+ return qryTimeout;
+
return IgniteSystemProperties.getLong(IGNITE_SQL_RETRY_TIMEOUT, DFLT_RETRY_TIMEOUT);
}
[15/50] [abbrv] ignite git commit: IGNITE-6846 Add metrics for entry
processor invocations. - Fixes #3148.
Posted by av...@apache.org.
IGNITE-6846 Add metrics for entry processor invocations. - Fixes #3148.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5e8669af
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5e8669af
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5e8669af
Branch: refs/heads/ignite-8446
Commit: 5e8669af709e8a4bfd41997212ee471efe4e6f36
Parents: bcda7a1
Author: voipp <al...@gmail.com>
Authored: Tue Jul 31 16:01:39 2018 +0300
Committer: Dmitriy Pavlov <dp...@apache.org>
Committed: Tue Jul 31 16:01:39 2018 +0300
----------------------------------------------------------------------
.../org/apache/ignite/cache/CacheMetrics.java | 77 ++++
.../cache/CacheClusterMetricsMXBeanImpl.java | 55 +++
.../cache/CacheLocalMetricsMXBeanImpl.java | 55 +++
.../processors/cache/CacheMetricsImpl.java | 206 +++++++++
.../processors/cache/CacheMetricsSnapshot.java | 157 +++++++
.../processors/cache/GridCacheAdapter.java | 67 ++-
.../processors/cache/GridCacheEntryEx.java | 8 +-
.../processors/cache/GridCacheMapEntry.java | 115 +++--
.../cache/GridCacheUpdateAtomicResult.java | 15 +-
.../distributed/dht/GridDhtTxPrepareFuture.java | 5 +
.../GridDhtAtomicAbstractUpdateFuture.java | 8 +-
.../GridDhtAtomicAbstractUpdateRequest.java | 16 +-
.../dht/atomic/GridDhtAtomicCache.java | 63 ++-
.../GridDhtAtomicSingleUpdateRequest.java | 11 +-
.../dht/atomic/GridDhtAtomicUpdateRequest.java | 5 +-
.../distributed/near/GridNearAtomicCache.java | 14 +-
.../local/atomic/GridLocalAtomicCache.java | 116 ++++--
.../cache/transactions/IgniteTxAdapter.java | 2 +
.../platform/cache/PlatformCache.java | 11 +
.../cache/GridCacheAbstractMetricsSelfTest.java | 417 +++++++++++++++++++
.../processors/cache/GridCacheTestEntryEx.java | 8 +-
.../GridCacheNearAtomicMetricsSelfTest.java | 35 ++
.../near/GridCacheNearMetricsSelfTest.java | 388 +++++++++++++++++
.../platform/PlatformCacheWriteMetricsTask.java | 55 +++
.../IgniteCacheMetricsSelfTestSuite.java | 2 +
.../Cache/CacheMetricsTest.cs | 11 +
.../Apache.Ignite.Core/Cache/ICacheMetrics.cs | 88 ++++
.../Impl/Cache/CacheMetricsImpl.cs | 79 +++-
28 files changed, 1999 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java b/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
index e3e6446..d3a4c04 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
@@ -72,6 +72,83 @@ public interface CacheMetrics {
public long getCachePuts();
/**
+ * The total number of cache invocations, caused update.
+ *
+ * @return The number of invocation updates.
+ */
+ public long getEntryProcessorPuts();
+
+ /**
+ * The total number of cache invocations, caused removal.
+ *
+ * @return The number of invocation removals.
+ */
+ public long getEntryProcessorRemovals();
+
+ /**
+ * The total number of cache invocations, caused no updates.
+ *
+ * @return The number of read-only invocations.
+ */
+ public long getEntryProcessorReadOnlyInvocations();
+
+ /**
+ * The total number of cache invocations.
+ *
+ * @return The number of cache invocations.
+ */
+ public long getEntryProcessorInvocations();
+
+ /**
+ * The total number of invocations on keys, which exist in cache.
+ *
+ * @return The number of cache invocation hits.
+ */
+ public long getEntryProcessorHits();
+
+ /**
+ * The percentage of invocations on keys, which exist in cache.
+ *
+ * @return The percentage of successful invocation hits.
+ */
+ public float getEntryProcessorHitPercentage();
+
+ /**
+ * The total number of invocations on keys, which don't exist in cache.
+ *
+ * @return The number of cache invocation misses.
+ */
+ public long getEntryProcessorMisses();
+
+ /**
+ * The percentage of invocations on keys, which don't exist in cache.
+ *
+ * @return The percentage of invocation misses.
+ */
+ public float getEntryProcessorMissPercentage();
+
+ /**
+ * The mean time to execute cache invokes.
+ *
+ * @return The time in µs.
+ */
+ public float getEntryProcessorAverageInvocationTime();
+
+ /**
+ * So far, the minimum time to execute cache invokes.
+ *
+ * @return The time in µs.
+ */
+ public float getEntryProcessorMinInvocationTime();
+
+ /**
+ * So far, the maximum time to execute cache invokes.
+ *
+ * @return The time in µs.
+ */
+ public float getEntryProcessorMaxInvocationTime();
+
+ /**
* The total number of removals from the cache. This does not include evictions,
* where the cache itself initiates the removal to make space.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java
index 8935a98..cbd0b57 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java
@@ -290,6 +290,61 @@ class CacheClusterMetricsMXBeanImpl implements CacheMetricsMXBean {
}
/** {@inheritDoc} */
+ @Override public long getEntryProcessorPuts() {
+ return cache.clusterMetrics().getEntryProcessorPuts();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorRemovals() {
+ return cache.clusterMetrics().getEntryProcessorRemovals();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorReadOnlyInvocations() {
+ return cache.clusterMetrics().getEntryProcessorReadOnlyInvocations();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorInvocations() {
+ return cache.clusterMetrics().getEntryProcessorInvocations();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorHits() {
+ return cache.clusterMetrics().getEntryProcessorHits();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorHitPercentage() {
+ return cache.clusterMetrics().getEntryProcessorHitPercentage();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMissPercentage() {
+ return cache.clusterMetrics().getEntryProcessorMissPercentage();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorMisses() {
+ return cache.clusterMetrics().getEntryProcessorMisses();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorAverageInvocationTime() {
+ return cache.clusterMetrics().getEntryProcessorAverageInvocationTime();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMinInvocationTime() {
+ return cache.clusterMetrics().getEntryProcessorMinInvocationTime();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMaxInvocationTime() {
+ return cache.clusterMetrics().getEntryProcessorMaxInvocationTime();
+ }
+
+ /** {@inheritDoc} */
@Override public long getCacheRemovals() {
return cache.clusterMetrics().getCacheRemovals();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java
index 212c7a0..790fe00 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java
@@ -286,6 +286,61 @@ class CacheLocalMetricsMXBeanImpl implements CacheMetricsMXBean {
}
/** {@inheritDoc} */
+ @Override public long getEntryProcessorPuts() {
+ return cache.metrics0().getEntryProcessorPuts();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorRemovals() {
+ return cache.metrics0().getEntryProcessorRemovals();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorReadOnlyInvocations() {
+ return cache.metrics0().getEntryProcessorReadOnlyInvocations();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorInvocations() {
+ return cache.metrics0().getEntryProcessorInvocations();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorHits() {
+ return cache.metrics0().getEntryProcessorHits();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorHitPercentage() {
+ return cache.metrics0().getEntryProcessorHitPercentage();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMissPercentage() {
+ return cache.metrics0().getEntryProcessorMissPercentage();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorMisses() {
+ return cache.metrics0().getEntryProcessorMisses();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorAverageInvocationTime() {
+ return cache.metrics0().getEntryProcessorAverageInvocationTime();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMinInvocationTime() {
+ return cache.metrics0().getEntryProcessorMinInvocationTime();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMaxInvocationTime() {
+ return cache.metrics0().getEntryProcessorMaxInvocationTime();
+ }
+
+ /** {@inheritDoc} */
@Override public long getCacheRemovals() {
return cache.metrics0().getCacheRemovals();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
index 0f6d06f..e81e995 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
@@ -51,6 +51,30 @@ public class CacheMetricsImpl implements CacheMetrics {
/** Number of reads. */
private AtomicLong reads = new AtomicLong();
+ /** Number of invocations caused update. */
+ private AtomicLong entryProcessorPuts = new AtomicLong();
+
+ /** Number of invocations caused removal. */
+ private AtomicLong entryProcessorRemovals = new AtomicLong();
+
+ /** Number of invocations caused update. */
+ private AtomicLong entryProcessorReadOnlyInvocations = new AtomicLong();
+
+ /** Entry processor invoke time taken nanos. */
+ private AtomicLong entryProcessorInvokeTimeNanos = new AtomicLong();
+
+ /** So far, the minimum time to execute cache invokes. */
+ private AtomicLong entryProcessorMinInvocationTime = new AtomicLong();
+
+ /** So far, the maximum time to execute cache invokes. */
+ private AtomicLong entryProcessorMaxInvocationTime = new AtomicLong();
+
+ /** Number of entry processor invokes on keys, which exist in cache. */
+ private AtomicLong entryProcessorHits = new AtomicLong();
+
+ /** Number of entry processor invokes on keys, which don't exist in cache. */
+ private AtomicLong entryProcessorMisses = new AtomicLong();
+
/** Number of writes. */
private AtomicLong writes = new AtomicLong();
@@ -439,6 +463,15 @@ public class CacheMetricsImpl implements CacheMetrics {
commitTimeNanos.set(0);
rollbackTimeNanos.set(0);
+ entryProcessorPuts.set(0);
+ entryProcessorRemovals.set(0);
+ entryProcessorReadOnlyInvocations.set(0);
+ entryProcessorMisses.set(0);
+ entryProcessorHits.set(0);
+ entryProcessorInvokeTimeNanos.set(0);
+ entryProcessorMaxInvocationTime.set(0);
+ entryProcessorMinInvocationTime.set(0);
+
offHeapGets.set(0);
offHeapPuts.set(0);
offHeapRemoves.set(0);
@@ -495,6 +528,79 @@ public class CacheMetricsImpl implements CacheMetrics {
}
/** {@inheritDoc} */
+ @Override public long getEntryProcessorPuts() {
+ return entryProcessorPuts.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorRemovals() {
+ return entryProcessorRemovals.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorReadOnlyInvocations() {
+ return entryProcessorReadOnlyInvocations.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorInvocations() {
+ return entryProcessorReadOnlyInvocations.get() + entryProcessorPuts.get() + entryProcessorRemovals.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorHits() {
+ return entryProcessorHits.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorHitPercentage() {
+ long hits = entryProcessorHits.get();
+ long totalInvocations = getEntryProcessorInvocations();
+
+ if (hits == 0)
+ return 0;
+
+ return (float) hits / totalInvocations * 100.0f;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorMisses() {
+ return entryProcessorMisses.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMissPercentage() {
+ long misses = entryProcessorMisses.get();
+ long totalInvocations = getEntryProcessorInvocations();
+
+ if (misses == 0)
+ return 0;
+
+ return (float) misses / totalInvocations * 100.0f;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorAverageInvocationTime() {
+ long totalInvokes = getEntryProcessorInvocations();
+ long timeNanos = entryProcessorInvokeTimeNanos.get();
+
+ if (timeNanos == 0 || totalInvokes == 0)
+ return 0;
+
+ return (1f * timeNanos) / totalInvokes / NANOS_IN_MICROSECOND;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMinInvocationTime() {
+ return (1f * entryProcessorMinInvocationTime.get()) / NANOS_IN_MICROSECOND;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMaxInvocationTime() {
+ return (1f * entryProcessorMaxInvocationTime.get()) / NANOS_IN_MICROSECOND;
+ }
+
+ /** {@inheritDoc} */
@Override public long getCacheRemovals() {
return rmCnt.get();
}
@@ -554,6 +660,106 @@ public class CacheMetricsImpl implements CacheMetrics {
}
/**
+ * Cache invocations caused update callback.
+ *
+ * @param isHit Hit or miss flag.
+ */
+ public void onInvokeUpdate(boolean isHit) {
+ entryProcessorPuts.incrementAndGet();
+
+ if (isHit)
+ entryProcessorHits.incrementAndGet();
+ else
+ entryProcessorMisses.incrementAndGet();
+
+ if (delegate != null)
+ delegate.onInvokeUpdate(isHit);
+ }
+
+ /**
+ * Cache invocations caused removal callback.
+ *
+ * @param isHit Hit or miss flag.
+ */
+ public void onInvokeRemove(boolean isHit) {
+ entryProcessorRemovals.incrementAndGet();
+
+ if (isHit)
+ entryProcessorHits.incrementAndGet();
+ else
+ entryProcessorMisses.incrementAndGet();
+
+ if (delegate != null)
+ delegate.onInvokeRemove(isHit);
+ }
+
+ /**
+ * Read-only cache invocations.
+ *
+ * @param isHit Hit or miss flag.
+ */
+ public void onReadOnlyInvoke(boolean isHit) {
+ entryProcessorReadOnlyInvocations.incrementAndGet();
+
+ if (isHit)
+ entryProcessorHits.incrementAndGet();
+ else
+ entryProcessorMisses.incrementAndGet();
+
+ if (delegate != null)
+ delegate.onReadOnlyInvoke(isHit);
+ }
+
+ /**
+ * Increments invoke operation time nanos.
+ *
+ * @param duration Duration.
+ */
+ public void addInvokeTimeNanos(long duration) {
+ entryProcessorInvokeTimeNanos.addAndGet(duration);
+
+ recalculateInvokeMinTimeNanos(duration);
+
+ recalculateInvokeMaxTimeNanos(duration);
+
+ if (delegate != null)
+ delegate.addInvokeTimeNanos(duration);
+
+ }
+
+ /**
+ * Recalculates invoke operation minimum time nanos.
+ *
+ * @param duration Duration.
+ */
+ private void recalculateInvokeMinTimeNanos(long duration){
+ long minTime = entryProcessorMinInvocationTime.longValue();
+
+ while (minTime > duration || minTime == 0) {
+ if (entryProcessorMinInvocationTime.compareAndSet(minTime, duration))
+ break;
+ else
+ minTime = entryProcessorMinInvocationTime.longValue();
+ }
+ }
+
+ /**
+ * Recalculates invoke operation maximum time nanos.
+ *
+ * @param duration Duration.
+ */
+ private void recalculateInvokeMaxTimeNanos(long duration){
+ long maxTime = entryProcessorMaxInvocationTime.longValue();
+
+ while (maxTime < duration) {
+ if (entryProcessorMaxInvocationTime.compareAndSet(maxTime, duration))
+ break;
+ else
+ maxTime = entryProcessorMaxInvocationTime.longValue();
+ }
+ }
+
+ /**
* Cache write callback.
*/
public void onWrite() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java
index 8a0f0e4..5f3001c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java
@@ -38,6 +38,57 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable {
/** Number of puts. */
private long puts = 0;
+ /** Number of invokes caused updates. */
+ private long entryProcessorPuts = 0;
+
+ /** Number of invokes caused no updates. */
+ private long entryProcessorReadOnlyInvocations = 0;
+
+ /**
+ * The mean time to execute cache invokes
+ */
+ private float entryProcessorAverageInvocationTime = 0;
+
+ /**
+ * The total number of cache invocations.
+ */
+ private long entryProcessorInvocations = 0;
+
+ /**
+ * The total number of cache invocations, caused removal.
+ */
+ private long entryProcessorRemovals = 0;
+
+ /**
+ * The total number of invocations on keys, which don't exist in cache.
+ */
+ private long entryProcessorMisses = 0;
+
+ /**
+ * The total number of invocations on keys, which exist in cache.
+ */
+ private long entryProcessorHits = 0;
+
+ /**
+ * The percentage of invocations on keys, which don't exist in cache.
+ */
+ private float entryProcessorMissPercentage = 0;
+
+ /**
+ * The percentage of invocations on keys, which exist in cache.
+ */
+ private float entryProcessorHitPercentage = 0;
+
+ /**
+ * So far, the maximum time to execute cache invokes.
+ */
+ private float entryProcessorMaxInvocationTime = 0;
+
+ /**
+ * So far, the minimum time to execute cache invokes.
+ */
+ private float entryProcessorMinInvocationTime = 0;
+
/** Number of hits. */
private long hits = 0;
@@ -270,6 +321,18 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable {
evicts = m.getCacheEvictions();
removes = m.getCacheRemovals();
+ entryProcessorPuts = m.getEntryProcessorPuts();
+ entryProcessorReadOnlyInvocations = m.getEntryProcessorReadOnlyInvocations();
+ entryProcessorInvocations = m.getEntryProcessorInvocations();
+ entryProcessorRemovals = m.getEntryProcessorRemovals();
+ entryProcessorMisses = m.getEntryProcessorMisses();
+ entryProcessorHits = m.getEntryProcessorHits();
+ entryProcessorMissPercentage = m.getEntryProcessorMissPercentage();
+ entryProcessorHitPercentage = m.getEntryProcessorHitPercentage();
+ entryProcessorAverageInvocationTime = m.getEntryProcessorAverageInvocationTime();
+ entryProcessorMaxInvocationTime = m.getEntryProcessorMaxInvocationTime();
+ entryProcessorMinInvocationTime = m.getEntryProcessorMinInvocationTime();
+
putAvgTimeNanos = m.getAveragePutTime();
getAvgTimeNanos = m.getAverageGetTime();
rmvAvgTimeNanos = m.getAverageRemoveTime();
@@ -386,6 +449,18 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable {
evicts += e.getCacheEvictions();
removes += e.getCacheRemovals();
+ entryProcessorPuts = e.getEntryProcessorPuts();
+ entryProcessorReadOnlyInvocations = e.getEntryProcessorReadOnlyInvocations();
+ entryProcessorInvocations = e.getEntryProcessorInvocations();
+ entryProcessorRemovals = e.getEntryProcessorRemovals();
+ entryProcessorMisses = e.getEntryProcessorMisses();
+ entryProcessorHits = e.getEntryProcessorHits();
+ entryProcessorMissPercentage = e.getEntryProcessorMissPercentage();
+ entryProcessorHitPercentage = e.getEntryProcessorHitPercentage();
+ entryProcessorAverageInvocationTime = e.getEntryProcessorAverageInvocationTime();
+ entryProcessorMaxInvocationTime = e.getEntryProcessorMaxInvocationTime();
+ entryProcessorMinInvocationTime = e.getEntryProcessorMinInvocationTime();
+
putAvgTimeNanos += e.getAveragePutTime();
getAvgTimeNanos += e.getAverageGetTime();
rmvAvgTimeNanos += e.getAverageRemoveTime();
@@ -524,6 +599,61 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable {
}
/** {@inheritDoc} */
+ @Override public long getEntryProcessorPuts() {
+ return entryProcessorPuts;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorReadOnlyInvocations() {
+ return entryProcessorReadOnlyInvocations;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorInvocations() {
+ return entryProcessorInvocations;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorHits() {
+ return entryProcessorHits;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorHitPercentage() {
+ return entryProcessorHitPercentage;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMissPercentage() {
+ return entryProcessorMissPercentage;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorMisses() {
+ return entryProcessorMisses;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorRemovals() {
+ return entryProcessorRemovals;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorAverageInvocationTime() {
+ return entryProcessorAverageInvocationTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMinInvocationTime() {
+ return entryProcessorMinInvocationTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMaxInvocationTime() {
+ return entryProcessorMaxInvocationTime;
+ }
+
+ /** {@inheritDoc} */
@Override public long getCacheRemovals() {
return removes;
}
@@ -950,6 +1080,18 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable {
out.writeLong(rebalanceStartTime);
out.writeLong(rebalanceFinishTime);
out.writeLong(rebalanceClearingPartitionsLeft);
+
+ out.writeLong(entryProcessorPuts);
+ out.writeFloat(entryProcessorAverageInvocationTime);
+ out.writeLong(entryProcessorInvocations);
+ out.writeFloat(entryProcessorMaxInvocationTime);
+ out.writeFloat(entryProcessorMinInvocationTime);
+ out.writeLong(entryProcessorReadOnlyInvocations);
+ out.writeFloat(entryProcessorHitPercentage);
+ out.writeLong(entryProcessorHits);
+ out.writeLong(entryProcessorMisses);
+ out.writeFloat(entryProcessorMissPercentage);
+ out.writeLong(entryProcessorRemovals);
}
/** {@inheritDoc} */
@@ -1011,5 +1153,20 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable {
rebalanceStartTime = in.readLong();
rebalanceFinishTime = in.readLong();
rebalanceClearingPartitionsLeft = in.readLong();
+
+ // 11 long and 5 float values give 108 bytes in total.
+ if (in.available() >= 108) {
+ entryProcessorPuts = in.readLong();
+ entryProcessorAverageInvocationTime = in.readFloat();
+ entryProcessorInvocations = in.readLong();
+ entryProcessorMaxInvocationTime = in.readFloat();
+ entryProcessorMinInvocationTime = in.readFloat();
+ entryProcessorReadOnlyInvocations = in.readLong();
+ entryProcessorHitPercentage = in.readFloat();
+ entryProcessorHits = in.readLong();
+ entryProcessorMisses = in.readLong();
+ entryProcessorMissPercentage = in.readFloat();
+ entryProcessorRemovals = in.readLong();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index c59e84e..eeed4fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -2542,6 +2542,10 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (topVer != null)
tx.topologyVersion(topVer);
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
IgniteInternalFuture<GridCacheReturn> fut = tx.invokeAsync(ctx,
null,
key,
@@ -2550,6 +2554,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
Map<K, EntryProcessorResult<T>> resMap = fut.get().value();
+ if (statsEnabled)
+ metrics0().addInvokeTimeNanos(System.nanoTime() - start);
+
EntryProcessorResult<T> res = null;
if (resMap != null) {
@@ -2572,6 +2579,10 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKeys(keys);
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
return syncOp(new SyncOp<Map<K, EntryProcessorResult<T>>>(keys.size() == 1) {
@Override public Map<K, EntryProcessorResult<T>> op(GridNearTxLocal tx)
throws IgniteCheckedException {
@@ -2586,6 +2597,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
Map<K, EntryProcessorResult<T>> res = fut.get().value();
+ if (statsEnabled)
+ metrics0().addInvokeTimeNanos(System.nanoTime() - start);
+
return res != null ? res : Collections.<K, EntryProcessorResult<T>>emptyMap();
}
});
@@ -2602,6 +2616,10 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKey(key);
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
IgniteInternalFuture<?> fut = asyncOp(new AsyncOp() {
@Override public IgniteInternalFuture op(GridNearTxLocal tx, AffinityTopologyVersion readyTopVer) {
Map<? extends K, EntryProcessor<K, V, Object>> invokeMap =
@@ -2624,6 +2642,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
throws IgniteCheckedException {
GridCacheReturn ret = fut.get();
+ if (statsEnabled)
+ metrics0().addInvokeTimeNanos(System.nanoTime() - start);
+
Map<K, EntryProcessorResult<T>> resMap = ret.value();
if (resMap != null) {
@@ -2647,6 +2668,10 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKeys(keys);
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
IgniteInternalFuture<?> fut = asyncOp(new AsyncOp(keys) {
@Override public IgniteInternalFuture<GridCacheReturn> op(GridNearTxLocal tx,
AffinityTopologyVersion readyTopVer) {
@@ -2674,6 +2699,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
throws IgniteCheckedException {
GridCacheReturn ret = fut.get();
+ if (statsEnabled)
+ metrics0().addInvokeTimeNanos(System.nanoTime() - start);
+
assert ret != null;
return ret.value() != null ? ret.<Map<K, EntryProcessorResult<T>>>value() : Collections.<K, EntryProcessorResult<T>>emptyMap();
@@ -2690,6 +2718,10 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKeys(map.keySet());
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
IgniteInternalFuture<?> fut = asyncOp(new AsyncOp(map.keySet()) {
@Override public IgniteInternalFuture<GridCacheReturn> op(GridNearTxLocal tx,
AffinityTopologyVersion readyTopVer) {
@@ -2712,6 +2744,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
throws IgniteCheckedException {
GridCacheReturn ret = fut.get();
+ if (statsEnabled)
+ metrics0().addInvokeTimeNanos(System.nanoTime() - start);
+
assert ret != null;
return ret.value() != null ? ret.<Map<K, EntryProcessorResult<T>>>value() : Collections.<K, EntryProcessorResult<T>>emptyMap();
@@ -2728,13 +2763,22 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKeys(map.keySet());
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
return syncOp(new SyncOp<Map<K, EntryProcessorResult<T>>>(map.size() == 1) {
@Nullable @Override public Map<K, EntryProcessorResult<T>> op(GridNearTxLocal tx)
throws IgniteCheckedException {
IgniteInternalFuture<GridCacheReturn> fut =
tx.invokeAsync(ctx, null, (Map<? extends K, ? extends EntryProcessor<K, V, Object>>)map, args);
- return fut.get().value();
+ Map<K, EntryProcessorResult<T>> value = fut.get().value();
+
+ if (statsEnabled)
+ metrics0().addInvokeTimeNanos(System.nanoTime() - start);
+
+ return value;
}
});
}
@@ -6192,6 +6236,27 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/**
+ *
+ */
+ protected static class InvokeAllTimeStatClosure<T> extends UpdateTimeStatClosure {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * @param metrics Metrics.
+ * @param start Start time.
+ */
+ public InvokeAllTimeStatClosure(CacheMetricsImpl metrics, final long start) {
+ super(metrics, start);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void updateTimeStat() {
+ metrics.addInvokeTimeNanos(System.nanoTime() - start);
+ }
+ }
+
+ /**
* Delayed callable class.
*/
public static abstract class TopologyVersionAwareJob extends ComputeJobAdapter {
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index fc374bb..aef38e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -450,6 +450,7 @@ public interface GridCacheEntryEx {
* @param taskName Task name.
* @param updateCntr Update counter.
* @param fut Dht atomic future.
+ * @param transformOp {@code True} if transform operation caused update.
* @return Tuple where first value is flag showing whether operation succeeded,
* second value is old entry value if return value is requested, third is updated entry value,
* fourth is the version to enqueue for deferred delete the fifth is DR conflict context
@@ -486,7 +487,8 @@ public interface GridCacheEntryEx {
String taskName,
@Nullable CacheObject prevVal,
@Nullable Long updateCntr,
- @Nullable GridDhtAtomicAbstractUpdateFuture fut
+ @Nullable GridDhtAtomicAbstractUpdateFuture fut,
+ boolean transformOp
) throws IgniteCheckedException, GridCacheEntryRemovedException;
/**
@@ -506,6 +508,7 @@ public interface GridCacheEntryEx {
* @param intercept If {@code true} then calls cache interceptor.
* @param subjId Subject ID initiated this update.
* @param taskName Task name.
+ * @param transformOp {@code True} if transform operation caused update.
* @return Tuple containing success flag, old value and result for invoke operation.
* @throws IgniteCheckedException If update failed.
* @throws GridCacheEntryRemovedException If entry is obsolete.
@@ -525,7 +528,8 @@ public interface GridCacheEntryEx {
@Nullable CacheEntryPredicate[] filter,
boolean intercept,
@Nullable UUID subjId,
- String taskName
+ String taskName,
+ boolean transformOp
) throws IgniteCheckedException, GridCacheEntryRemovedException;
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 03f6d11..9f9b5c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -74,6 +74,7 @@ import org.apache.ignite.internal.util.lang.GridTuple3;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.T3;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -89,8 +90,11 @@ import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_READ;
import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_REMOVED;
import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_UNLOCKED;
import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.READ;
import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM;
import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE;
+import static org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult.UpdateOutcome.INVOKE_NO_OP;
+import static org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult.UpdateOutcome.REMOVE_NO_VAL;
import static org.apache.ignite.internal.processors.dr.GridDrType.DR_NONE;
/**
@@ -1045,9 +1049,15 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
recordNodeId(affNodeId, topVer);
- if (metrics && cctx.statisticsEnabled())
+ if (metrics && cctx.statisticsEnabled()) {
cctx.cache().metrics0().onWrite();
+ T2<GridCacheOperation, CacheObject> entryProcRes = tx.entry(txKey()).entryProcessorCalculatedValue();
+
+ if (entryProcRes != null && UPDATE.equals(entryProcRes.get1()))
+ cctx.cache().metrics0().onInvokeUpdate(old != null);
+ }
+
if (evt && newVer != null && cctx.events().isRecordable(EVT_CACHE_OBJECT_PUT)) {
CacheObject evtOld = cctx.unwrapTemporary(old);
@@ -1230,9 +1240,15 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
drReplicate(drType, null, newVer, topVer);
- if (metrics && cctx.statisticsEnabled())
+ if (metrics && cctx.statisticsEnabled()) {
cctx.cache().metrics0().onRemove();
+ T2<GridCacheOperation, CacheObject> entryProcRes = tx.entry(txKey()).entryProcessorCalculatedValue();
+
+ if (entryProcRes != null && DELETE.equals(entryProcRes.get1()))
+ cctx.cache().metrics0().onInvokeRemove(old != null);
+ }
+
if (tx == null)
obsoleteVer = newVer;
else {
@@ -1358,7 +1374,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
@Nullable CacheEntryPredicate[] filter,
boolean intercept,
@Nullable UUID subjId,
- String taskName
+ String taskName,
+ boolean transformOp
) throws IgniteCheckedException, GridCacheEntryRemovedException {
assert cctx.isLocal() && cctx.atomic();
@@ -1479,6 +1496,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
try {
Object computed = entryProcessor.process(entry, invokeArgs);
+ transformOp = true;
+
if (entry.modified()) {
updated0 = cctx.unwrapTemporary(entry.getValue());
@@ -1503,6 +1522,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (expiryPlc != null && !readFromStore && hasValueUnlocked())
updateTtl(expiryPlc);
+ updateMetrics(READ, metrics, transformOp, old != null);
+
return new GridTuple3<>(false, null, invokeRes);
}
}
@@ -1564,9 +1585,13 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
}
- if (ttl == CU.TTL_ZERO)
+ if (ttl == CU.TTL_ZERO) {
op = GridCacheOperation.DELETE;
+ //If time expired no transformation needed.
+ transformOp = false;
+ }
+
// Try write-through.
if (op == GridCacheOperation.UPDATE) {
// Detach value before index update.
@@ -1634,7 +1659,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
if (res)
- updateMetrics(op, metrics);
+ updateMetrics(op, metrics, transformOp, old != null);
+ else if (op == DELETE && transformOp)
+ cctx.cache().metrics0().onInvokeRemove(old != null);
if (lsnrCol != null) {
long updateCntr = nextPartitionCounter(AffinityTopologyVersion.NONE, true, null);
@@ -1705,7 +1732,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
final String taskName,
@Nullable final CacheObject prevVal,
@Nullable final Long updateCntr,
- @Nullable final GridDhtAtomicAbstractUpdateFuture fut
+ @Nullable final GridDhtAtomicAbstractUpdateFuture fut,
+ boolean transformOp
) throws IgniteCheckedException, GridCacheEntryRemovedException, GridClosureException {
assert cctx.atomic() && !detached();
@@ -1769,13 +1797,20 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CacheObject updateVal = null;
GridCacheVersion updateVer = c.newVer;
+ boolean updateMetrics = metrics && cctx.statisticsEnabled();
+
// Apply metrics.
- if (metrics &&
+ if (updateMetrics &&
updateRes.outcome().updateReadMetrics() &&
- cctx.statisticsEnabled() &&
needVal)
cctx.cache().metrics0().onRead(oldVal != null);
+ if (updateMetrics && INVOKE_NO_OP.equals(updateRes.outcome()) && (transformOp || updateRes.transformed()))
+ cctx.cache().metrics0().onReadOnlyInvoke(oldVal != null);
+ else if (updateMetrics && REMOVE_NO_VAL.equals(updateRes.outcome())
+ && (transformOp || updateRes.transformed()))
+ cctx.cache().metrics0().onInvokeRemove(oldVal != null);
+
switch (updateRes.outcome()) {
case VERSION_CHECK_FAILED: {
if (!cctx.isNear()) {
@@ -1913,7 +1948,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
if (updateRes.success())
- updateMetrics(c.op, metrics);
+ updateMetrics(c.op, metrics, transformOp || updateRes.transformed(), oldVal != null);
// Continuous query filter should be perform under lock.
if (lsnrs != null) {
@@ -4225,13 +4260,24 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
*
* @param op Operation.
* @param metrics Update merics flag.
+ * @param transformed {@code True} if transform operation caused update.
+ * @param hasOldVal {@code True} if entry has old value.
*/
- private void updateMetrics(GridCacheOperation op, boolean metrics) {
+ private void updateMetrics(GridCacheOperation op, boolean metrics, boolean transformed, boolean hasOldVal) {
if (metrics && cctx.statisticsEnabled()) {
- if (op == GridCacheOperation.DELETE)
+ if (op == GridCacheOperation.DELETE) {
cctx.cache().metrics0().onRemove();
- else
+
+ if (transformed)
+ cctx.cache().metrics0().onInvokeRemove(hasOldVal);
+ } else if (op == READ && transformed)
+ cctx.cache().metrics0().onReadOnlyInvoke(hasOldVal);
+ else {
cctx.cache().metrics0().onWrite();
+
+ if (transformed)
+ cctx.cache().metrics0().onInvokeUpdate(hasOldVal);
+ }
}
}
@@ -4699,12 +4745,16 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
boolean invoke = op == TRANSFORM;
+ boolean transformed = false;
+
if (invoke) {
invokeEntry = new CacheInvokeEntry<>(entry.key, oldVal, entry.ver, keepBinary, entry);
invokeRes = runEntryProcessor(invokeEntry);
op = writeObj == null ? DELETE : UPDATE;
+
+ transformed = true;
}
CacheObject newVal = (CacheObject)writeObj;
@@ -4747,7 +4797,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- 0);
+ 0,
+ false);
return;
}
@@ -4765,7 +4816,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- 0);
+ 0,
+ true);
return;
}
@@ -4784,7 +4836,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- 0);
+ 0,
+ false);
return;
}
@@ -4805,12 +4858,12 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (op == UPDATE) {
assert writeObj != null;
- update(conflictCtx, invokeRes, storeLoadedVal != null);
+ update(conflictCtx, invokeRes, storeLoadedVal != null, transformed);
}
else {
assert op == DELETE && writeObj == null : op;
- remove(conflictCtx, invokeRes, storeLoadedVal != null);
+ remove(conflictCtx, invokeRes, storeLoadedVal != null, transformed);
}
assert updateRes != null && treeOp != null;
@@ -4934,11 +4987,13 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
* @param conflictCtx Conflict context.
* @param invokeRes Entry processor result (for invoke operation).
* @param readFromStore {@code True} if initial entry value was {@code null} and it was read from store.
+ * @param transformed {@code True} if update caused by transformation operation.
* @throws IgniteCheckedException If failed.
*/
private void update(@Nullable GridCacheVersionConflictContext<?, ?> conflictCtx,
@Nullable IgniteBiTuple<Object, Exception> invokeRes,
- boolean readFromStore)
+ boolean readFromStore,
+ boolean transformed)
throws IgniteCheckedException
{
GridCacheContext cctx = entry.context();
@@ -4979,7 +5034,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
writeObj = null;
- remove(conflictCtx, invokeRes, readFromStore);
+ remove(conflictCtx, invokeRes, readFromStore, false);
return;
}
@@ -5018,7 +5073,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- 0);
+ 0,
+ false);
return;
}
@@ -5081,19 +5137,22 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
newSysExpireTime,
null,
conflictCtx,
- updateCntr0);
+ updateCntr0,
+ transformed);
}
/**
* @param conflictCtx Conflict context.
* @param invokeRes Entry processor result (for invoke operation).
* @param readFromStore {@code True} if initial entry value was {@code null} and it was read from store.
+ * @param transformed {@code True} if remove caused by tranformation operation.
* @throws IgniteCheckedException If failed.
*/
@SuppressWarnings("unchecked")
private void remove(@Nullable GridCacheVersionConflictContext<?, ?> conflictCtx,
@Nullable IgniteBiTuple<Object, Exception> invokeRes,
- boolean readFromStore)
+ boolean readFromStore,
+ boolean transformed)
throws IgniteCheckedException
{
GridCacheContext cctx = entry.context();
@@ -5123,7 +5182,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- 0);
+ 0,
+ false);
return;
}
@@ -5178,7 +5238,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_CALCULATE,
enqueueVer,
conflictCtx,
- updateCntr0);
+ updateCntr0,
+ transformed);
}
/**
@@ -5259,7 +5320,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- 0);
+ 0,
+ false);
}
// Will update something.
else {
@@ -5325,7 +5387,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- 0);
+ 0,
+ false);
}
}
else
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateAtomicResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateAtomicResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateAtomicResult.java
index 97cb534..0bf1c57 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateAtomicResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateAtomicResult.java
@@ -57,6 +57,9 @@ public class GridCacheUpdateAtomicResult {
/** */
private final long updateCntr;
+ /** Flag indicating whether value is transformed. */
+ private boolean transformed;
+
/** Value computed by entry processor. */
private IgniteBiTuple<Object, Exception> res;
@@ -72,6 +75,7 @@ public class GridCacheUpdateAtomicResult {
* @param rmvVer Version for deferred delete.
* @param conflictRes DR resolution result.
* @param updateCntr Partition update counter.
+ * @param transformed {@code True} if result was transformed.
*/
GridCacheUpdateAtomicResult(UpdateOutcome outcome,
@Nullable CacheObject oldVal,
@@ -81,7 +85,8 @@ public class GridCacheUpdateAtomicResult {
long conflictExpireTime,
@Nullable GridCacheVersion rmvVer,
@Nullable GridCacheVersionConflictContext<?, ?> conflictRes,
- long updateCntr) {
+ long updateCntr,
+ boolean transformed) {
assert outcome != null;
this.outcome = outcome;
@@ -93,6 +98,7 @@ public class GridCacheUpdateAtomicResult {
this.rmvVer = rmvVer;
this.conflictRes = conflictRes;
this.updateCntr = updateCntr;
+ this.transformed = transformed;
}
/**
@@ -240,6 +246,13 @@ public class GridCacheUpdateAtomicResult {
}
}
+ /**
+ * @return {@code True} if transformed.
+ */
+ public boolean transformed() {
+ return transformed;
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(GridCacheUpdateAtomicResult.class, this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index d02b851..ba1210e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -440,6 +440,11 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
GridCacheOperation op = modified ? (val == null ? DELETE : UPDATE) : NOOP;
if (op == NOOP) {
+ GridCacheAdapter<?, ?> cache = writeEntry.context().cache();
+
+ if (cache.context().statisticsEnabled())
+ cache.metrics0().onReadOnlyInvoke(oldVal != null);
+
if (expiry != null) {
long ttl = CU.toTtl(expiry.getExpiryForAccess());
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateFuture.java
index e8824e7..c04e1af 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateFuture.java
@@ -38,6 +38,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheAtomicFuture;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
import org.apache.ignite.internal.processors.cache.GridCacheFutureAdapter;
+import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.GridCacheReturn;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
@@ -161,6 +162,7 @@ public abstract class GridDhtAtomicAbstractUpdateFuture extends GridCacheFutureA
* @param addPrevVal If {@code true} sends previous value to backups.
* @param prevVal Previous value.
* @param updateCntr Partition update counter.
+ * @param cacheOp Corresponding cache operation.
*/
@SuppressWarnings("ForLoopReplaceableByForEach")
final void addWriteEntry(
@@ -173,7 +175,8 @@ public abstract class GridDhtAtomicAbstractUpdateFuture extends GridCacheFutureA
@Nullable GridCacheVersion conflictVer,
boolean addPrevVal,
@Nullable CacheObject prevVal,
- long updateCntr) {
+ long updateCntr,
+ GridCacheOperation cacheOp) {
AffinityTopologyVersion topVer = updateReq.topologyVersion();
List<ClusterNode> affNodes = affAssignment.get(entry.partition());
@@ -222,7 +225,8 @@ public abstract class GridDhtAtomicAbstractUpdateFuture extends GridCacheFutureA
conflictVer,
addPrevVal,
prevVal,
- updateCntr);
+ updateCntr,
+ cacheOp);
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java
index a50b68c..a5e9feb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java
@@ -28,6 +28,7 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheDeployable;
import org.apache.ignite.internal.processors.cache.GridCacheIdMessage;
+import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -59,6 +60,9 @@ public abstract class GridDhtAtomicAbstractUpdateRequest extends GridCacheIdMess
/** */
protected static final int DHT_ATOMIC_OBSOLETE_NEAR_KEY_FLAG_MASK = 0x20;
+ /** Flag indicating transformation operation was performed. */
+ protected static final int DHT_ATOMIC_TRANSFORM_OP_FLAG_MASK = 0x40;
+
/** Message index. */
public static final int CACHE_MSG_IDX = nextIndexId();
@@ -225,6 +229,13 @@ public abstract class GridDhtAtomicAbstractUpdateRequest extends GridCacheIdMess
}
/**
+ * @return {@code True} if transformation operation was performed.
+ */
+ public final boolean transformOperation() {
+ return isFlag(DHT_ATOMIC_TRANSFORM_OP_FLAG_MASK);
+ }
+
+ /**
* @return {@code True} if on response flag changed.
*/
public boolean onResponse() {
@@ -268,6 +279,7 @@ public abstract class GridDhtAtomicAbstractUpdateRequest extends GridCacheIdMess
* @param addPrevVal If {@code true} adds previous value.
* @param prevVal Previous value.
* @param updateCntr Update counter.
+ * @param cacheOp Corresponding cache operation.
*/
public abstract void addWriteValue(KeyCacheObject key,
@Nullable CacheObject val,
@@ -277,8 +289,8 @@ public abstract class GridDhtAtomicAbstractUpdateRequest extends GridCacheIdMess
@Nullable GridCacheVersion conflictVer,
boolean addPrevVal,
@Nullable CacheObject prevVal,
- long updateCntr
- );
+ long updateCntr,
+ GridCacheOperation cacheOp);
/**
* @param key Key to add.
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 8408b32..c39842e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -820,6 +820,10 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
if (keyCheck)
validateCacheKey(key);
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
CacheOperationContext opCtx = ctx.operationContextPerCall();
final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
@@ -838,6 +842,9 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
throws IgniteCheckedException {
Map<K, EntryProcessorResult<T>> resMap = fut.get();
+ if (statsEnabled)
+ metrics0().addInvokeTimeNanos(System.nanoTime() - start);
+
if (resMap != null) {
assert resMap.isEmpty() || resMap.size() == 1 : resMap.size();
@@ -885,6 +892,10 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
if (keyCheck)
validateCacheKeys(keys);
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
Map<? extends K, EntryProcessor> invokeMap = F.viewAsMap(keys, new C1<K, EntryProcessor>() {
@Override public EntryProcessor apply(K k) {
return entryProcessor;
@@ -912,6 +923,9 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
) throws IgniteCheckedException {
Map<Object, EntryProcessorResult> resMap = (Map)fut.get();
+ if (statsEnabled)
+ metrics0().addInvokeTimeNanos(System.nanoTime() - start);
+
return ctx.unwrapInvokeResult(resMap, keepBinary);
}
});
@@ -926,7 +940,11 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
if (keyCheck)
validateCacheKeys(map.keySet());
- return (Map<K, EntryProcessorResult<T>>)updateAll0(null,
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
+ Map<K, EntryProcessorResult<T>> updateResults = (Map<K, EntryProcessorResult<T>>) updateAll0(null,
map,
args,
null,
@@ -935,6 +953,11 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
false,
TRANSFORM,
false).get();
+
+ if (statsEnabled)
+ metrics0().addInvokeTimeNanos(System.nanoTime() - start);
+
+ return updateResults;
}
/** {@inheritDoc} */
@@ -947,7 +970,11 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
if (keyCheck)
validateCacheKeys(map.keySet());
- return updateAll0(null,
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
+ IgniteInternalFuture updateResults = updateAll0(null,
map,
args,
null,
@@ -956,6 +983,11 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
false,
TRANSFORM,
true);
+
+ if (statsEnabled)
+ updateResults.listen(new InvokeAllTimeStatClosure(metrics0(), start));
+
+ return updateResults;
}
/**
@@ -2077,9 +2109,12 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
curInvokeRes = CacheInvokeResult.fromResult(computed);
}
- if (!invokeEntry.modified())
+ if (!invokeEntry.modified()) {
+ if (ctx.statisticsEnabled())
+ ctx.cache().metrics0().onReadOnlyInvoke(old != null);
+
continue;
- else {
+ } else {
updatedVal = ctx.unwrapTemporary(invokeEntry.getValue());
updated = ctx.toCacheObject(updatedVal);
@@ -2461,7 +2496,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
taskName,
/*prevVal*/null,
/*updateCntr*/null,
- dhtFut);
+ dhtFut,
+ false);
if (dhtFut != null) {
if (updRes.sendToDht()) { // Send to backups even in case of remove-remove scenarios.
@@ -2484,7 +2520,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
newConflictVer,
sndPrevVal,
updRes.oldValue(),
- updRes.updateCounter());
+ updRes.updateCounter(),
+ op);
if (readers != null)
dhtFut.addNearWriteEntries(
@@ -2687,6 +2724,9 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
// Get readers before innerUpdate (reader cleared after remove).
GridDhtCacheEntry.ReaderId[] readers = entry.readersLocked();
+ EntryProcessor<Object, Object, Object> entryProcessor =
+ entryProcessorMap == null ? null : entryProcessorMap.get(entry.key());
+
GridCacheUpdateAtomicResult updRes = entry.innerUpdate(
ver,
nearNode.id(),
@@ -2715,7 +2755,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
taskName,
null,
null,
- dhtFut);
+ dhtFut,
+ entryProcessor != null);
assert !updRes.success() || updRes.newTtl() == CU.TTL_NOT_CHANGED || expiry != null :
"success=" + updRes.success() + ", newTtl=" + updRes.newTtl() + ", expiry=" + expiry;
@@ -2740,8 +2781,6 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
batchRes.addDeleted(entry, updRes, entries);
if (dhtFut != null) {
- EntryProcessor<Object, Object, Object> entryProcessor =
- entryProcessorMap == null ? null : entryProcessorMap.get(entry.key());
dhtFut.addWriteEntry(
affAssignment,
@@ -2753,7 +2792,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
null,
sndPrevVal,
updRes.oldValue(),
- updRes.updateCounter());
+ updRes.updateCounter(),
+ op);
if (readers != null)
dhtFut.addNearWriteEntries(
@@ -3201,7 +3241,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
taskName,
prevVal,
updateIdx,
- null);
+ null,
+ req.transformOperation());
if (updRes.removeVersion() != null)
ctx.onDeferredDelete(entry, updRes.removeVersion());
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
index 0ade243..19b24b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
@@ -26,6 +26,7 @@ import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -38,6 +39,8 @@ import org.apache.ignite.plugin.extensions.communication.MessageWriter;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM;
+
/**
*
*/
@@ -118,6 +121,7 @@ public class GridDhtAtomicSingleUpdateRequest extends GridDhtAtomicAbstractUpdat
* @param addPrevVal If {@code true} adds previous value.
* @param prevVal Previous value.
* @param updateCntr Update counter.
+ * @param cacheOp Corresponding cache operation.
*/
@Override public void addWriteValue(KeyCacheObject key,
@Nullable CacheObject val,
@@ -127,8 +131,8 @@ public class GridDhtAtomicSingleUpdateRequest extends GridDhtAtomicAbstractUpdat
@Nullable GridCacheVersion conflictVer,
boolean addPrevVal,
@Nullable CacheObject prevVal,
- long updateCntr
- ) {
+ long updateCntr,
+ GridCacheOperation cacheOp) {
assert entryProcessor == null;
assert ttl <= 0 : ttl;
assert conflictExpireTime <= 0 : conflictExpireTime;
@@ -144,6 +148,9 @@ public class GridDhtAtomicSingleUpdateRequest extends GridDhtAtomicAbstractUpdat
this.prevVal = prevVal;
this.updateCntr = updateCntr;
+
+ if (cacheOp == TRANSFORM)
+ setFlag(true, DHT_ATOMIC_TRANSFORM_OP_FLAG_MASK);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
index 6f3f530..31439d6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
@@ -30,6 +30,7 @@ import org.apache.ignite.internal.GridDirectTransient;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -201,8 +202,8 @@ public class GridDhtAtomicUpdateRequest extends GridDhtAtomicAbstractUpdateReque
@Nullable GridCacheVersion conflictVer,
boolean addPrevVal,
@Nullable CacheObject prevVal,
- long updateCntr
- ) {
+ long updateCntr,
+ GridCacheOperation cacheOp) {
assert key.partition() >= 0 : key;
keys.add(key);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index 2f832b4..23c2480 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -197,7 +197,8 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
req.keepBinary(),
req.nodeId(),
req.subjectId(),
- taskName);
+ taskName,
+ req.operation() == TRANSFORM);
}
catch (IgniteCheckedException e) {
res.addFailedKey(key, new IgniteCheckedException("Failed to update key in near cache: " + key, e));
@@ -214,6 +215,7 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
* @param nodeId Node ID.
* @param subjId Subject ID.
* @param taskName Task name.
+ * @param transformedValue {@code True} if transformed value.
* @throws IgniteCheckedException If failed.
*/
private void processNearAtomicUpdateResponse(
@@ -225,8 +227,8 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
boolean keepBinary,
UUID nodeId,
UUID subjId,
- String taskName
- ) throws IgniteCheckedException {
+ String taskName,
+ boolean transformedValue) throws IgniteCheckedException {
try {
while (true) {
GridCacheEntryEx entry = null;
@@ -266,7 +268,8 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
taskName,
null,
null,
- null);
+ null,
+ transformedValue);
if (updRes.removeVersion() != null)
ctx.onDeferredDelete(entry, updRes.removeVersion());
@@ -365,7 +368,8 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
taskName,
null,
null,
- null);
+ null,
+ false);
if (updRes.removeVersion() != null)
ctx.onDeferredDelete(entry, updRes.removeVersion());
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index a9d7fa1..b96dbdc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -560,6 +560,10 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
if (keyCheck)
validateCacheKeys(keys);
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
Map<? extends K, EntryProcessor> invokeMap = F.viewAsMap(keys, new C1<K, EntryProcessor>() {
@Override public EntryProcessor apply(K k) {
return entryProcessor;
@@ -570,17 +574,23 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
- return (Map<K, EntryProcessorResult<T>>)updateAllInternal(TRANSFORM,
- invokeMap.keySet(),
- invokeMap.values(),
- args,
- expiryPerCall(),
- false,
- false,
- null,
- ctx.writeThrough(),
- ctx.readThrough(),
- keepBinary);
+ Map<K, EntryProcessorResult<T>> entryProcessorRes = (Map<K, EntryProcessorResult<T>>) updateAllInternal(
+ TRANSFORM,
+ invokeMap.keySet(),
+ invokeMap.values(),
+ args,
+ expiryPerCall(),
+ false,
+ false,
+ null,
+ ctx.writeThrough(),
+ ctx.readThrough(),
+ keepBinary);
+
+ if (statsEnabled)
+ metrics0().addInvokeTimeNanos(System.nanoTime() - start);
+
+ return entryProcessorRes;
}
/** {@inheritDoc} */
@@ -593,6 +603,10 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
if (keyCheck)
validateCacheKey(key);
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
Map<? extends K, EntryProcessor> invokeMap =
Collections.singletonMap(key, (EntryProcessor)entryProcessor);
@@ -608,6 +622,9 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
throws IgniteCheckedException {
Map<K, EntryProcessorResult<T>> resMap = fut.get();
+ if (statsEnabled)
+ metrics0().addInvokeTimeNanos(System.nanoTime() - start);
+
if (resMap != null) {
assert resMap.isEmpty() || resMap.size() == 1 : resMap.size();
@@ -630,18 +647,27 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
if (keyCheck)
validateCacheKeys(keys);
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
Map<? extends K, EntryProcessor> invokeMap = F.viewAsMap(keys, new C1<K, EntryProcessor>() {
@Override public EntryProcessor apply(K k) {
return entryProcessor;
}
});
- return updateAllAsync0(null,
+ IgniteInternalFuture fut = updateAllAsync0(null,
invokeMap,
args,
true,
false,
null);
+
+ if (statsEnabled)
+ fut.listen(new InvokeAllTimeStatClosure(metrics0(), start));
+
+ return fut;
}
/** {@inheritDoc} */
@@ -654,19 +680,29 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
if (keyCheck)
validateCacheKeys(map.keySet());
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
CacheOperationContext opCtx = ctx.operationContextPerCall();
- return (Map<K, EntryProcessorResult<T>>)updateAllInternal(TRANSFORM,
- map.keySet(),
- map.values(),
- args,
- expiryPerCall(),
- false,
- false,
- null,
- ctx.writeThrough(),
- ctx.readThrough(),
- opCtx != null && opCtx.isKeepBinary());
+ Map<K, EntryProcessorResult<T>> entryProcessorResult = (Map<K, EntryProcessorResult<T>>) updateAllInternal(
+ TRANSFORM,
+ map.keySet(),
+ map.values(),
+ args,
+ expiryPerCall(),
+ false,
+ false,
+ null,
+ ctx.writeThrough(),
+ ctx.readThrough(),
+ opCtx != null && opCtx.isKeepBinary());
+
+ if (statsEnabled)
+ metrics0().addInvokeTimeNanos(System.nanoTime() - start);
+
+ return entryProcessorResult;
}
/** {@inheritDoc} */
@@ -679,12 +715,21 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
if (keyCheck)
validateCacheKeys(map.keySet());
- return updateAllAsync0(null,
+ final boolean statsEnabled = ctx.statisticsEnabled();
+
+ final long start = statsEnabled ? System.nanoTime() : 0L;
+
+ IgniteInternalFuture fut = updateAllAsync0(null,
map,
args,
true,
false,
null);
+
+ if (statsEnabled)
+ fut.listen(new InvokeAllTimeStatClosure(metrics0(), start));
+
+ return fut;
}
/**
@@ -893,7 +938,8 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
filters,
intercept,
subjId,
- taskName);
+ taskName,
+ false);
if (op == TRANSFORM) {
if (t.get3() != null) {
@@ -1076,7 +1122,8 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
validation = true;
ctx.validateKeyAndValue(entry.key(), updated);
- }
+ } else if (ctx.statisticsEnabled() && !invokeEntry.modified())
+ ctx.cache().metrics0().onReadOnlyInvoke(old != null);
}
catch (Exception e) {
invokeRes = CacheInvokeResult.fromError(e);
@@ -1115,7 +1162,8 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
keepBinary,
err,
subjId,
- taskName);
+ taskName,
+ true);
putMap = null;
writeVals = null;
@@ -1153,7 +1201,8 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
keepBinary,
err,
subjId,
- taskName);
+ taskName,
+ true);
rmvKeys = null;
@@ -1258,7 +1307,8 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
keepBinary,
err,
subjId,
- taskName);
+ taskName,
+ op == TRANSFORM);
}
else
assert filtered.isEmpty();
@@ -1283,6 +1333,7 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
* @param err Optional partial update exception.
* @param subjId Subject ID.
* @param taskName Task name.
+ * @param transformed {@code True} if transform operation performed.
* @return Partial update exception.
*/
@SuppressWarnings({"unchecked", "ConstantConditions", "ForLoopReplaceableByForEach"})
@@ -1296,8 +1347,8 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
boolean keepBinary,
@Nullable CachePartialUpdateCheckedException err,
UUID subjId,
- String taskName
- ) {
+ String taskName,
+ boolean transformed) {
assert putMap == null ^ rmvKeys == null;
GridCacheOperation op;
@@ -1373,7 +1424,8 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
null,
false,
subjId,
- taskName);
+ taskName,
+ transformed);
if (intercept) {
if (op == UPDATE)
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 3cf1146..15df637 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -1578,6 +1578,8 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
GridCacheOperation op = modified ? (cacheVal == null ? DELETE : UPDATE) : NOOP;
+ txEntry.entryProcessorCalculatedValue(new T2<>(op, op == NOOP ? null : cacheVal));
+
if (op == NOOP) {
ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
[23/50] [abbrv] ignite git commit: IGNITE-8180 Fix for test
ZookeeperDiscoverySpiTest.testQuorumRestore fails on TC - Fixes #4203.
Posted by av...@apache.org.
IGNITE-8180 Fix for test ZookeeperDiscoverySpiTest.testQuorumRestore fails on TC - Fixes #4203.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ccd65e61
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ccd65e61
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ccd65e61
Branch: refs/heads/ignite-8446
Commit: ccd65e613e6927fc6dc1f0e9c99cfc86f792b0c4
Parents: 8bbcda3
Author: NSAmelchev <ns...@gmail.com>
Authored: Tue Jul 31 17:46:56 2018 +0300
Committer: Dmitriy Pavlov <dp...@apache.org>
Committed: Tue Jul 31 17:46:56 2018 +0300
----------------------------------------------------------------------
.../spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ccd65e61/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index f38a097..59577c8 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -1314,9 +1314,7 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
public void testQuorumRestore() throws Exception {
- fail("https://issues.apache.org/jira/browse/IGNITE-8180");
-
- sesTimeout = 15_000;
+ sesTimeout = 60_000;
startGrids(3);
[35/50] [abbrv] ignite git commit: IGNITE-8697 Flink sink throws
java.lang.IllegalArgumentException when running in flink cluster mode. -
Fixes #4398.
Posted by av...@apache.org.
IGNITE-8697 Flink sink throws java.lang.IllegalArgumentException when running in flink cluster mode. - Fixes #4398.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/891da2a5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/891da2a5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/891da2a5
Branch: refs/heads/ignite-8446
Commit: 891da2a5b61e2aa70a0decf5afca91f5294d94b9
Parents: 137dd06
Author: samaitra <sa...@gmail.com>
Authored: Wed Aug 1 21:02:02 2018 +0300
Committer: Dmitriy Pavlov <dp...@apache.org>
Committed: Wed Aug 1 21:02:02 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/sink/flink/IgniteSink.java | 88 +++++------
.../sink/flink/FlinkIgniteSinkSelfTest.java | 154 +++----------------
.../flink/src/test/resources/example-ignite.xml | 7 +-
3 files changed, 71 insertions(+), 178 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/891da2a5/modules/flink/src/main/java/org/apache/ignite/sink/flink/IgniteSink.java
----------------------------------------------------------------------
diff --git a/modules/flink/src/main/java/org/apache/ignite/sink/flink/IgniteSink.java b/modules/flink/src/main/java/org/apache/ignite/sink/flink/IgniteSink.java
index 2f18f80..ad29490 100644
--- a/modules/flink/src/main/java/org/apache/ignite/sink/flink/IgniteSink.java
+++ b/modules/flink/src/main/java/org/apache/ignite/sink/flink/IgniteSink.java
@@ -18,10 +18,12 @@
package org.apache.ignite.sink.flink;
import java.util.Map;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteIllegalStateException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.Ignition;
import org.apache.ignite.internal.util.typedef.internal.A;
@@ -34,7 +36,7 @@ public class IgniteSink<IN> extends RichSinkFunction<IN> {
private static final long DFLT_FLUSH_FREQ = 10000L;
/** Logger. */
- private final transient IgniteLogger log;
+ private transient IgniteLogger log;
/** Automatic flush frequency. */
private long autoFlushFrequency = DFLT_FLUSH_FREQ;
@@ -43,13 +45,19 @@ public class IgniteSink<IN> extends RichSinkFunction<IN> {
private boolean allowOverwrite = false;
/** Flag for stopped state. */
- private static volatile boolean stopped = true;
+ private volatile boolean stopped = true;
+
+ /** Ignite instance. */
+ protected transient Ignite ignite;
+
+ /** Ignite Data streamer instance. */
+ protected transient IgniteDataStreamer streamer;
/** Ignite grid configuration file. */
- private static String igniteCfgFile;
+ protected final String igniteCfgFile;
/** Cache name. */
- private static String cacheName;
+ protected final String cacheName;
/**
* Gets the cache name.
@@ -70,6 +78,15 @@ public class IgniteSink<IN> extends RichSinkFunction<IN> {
}
/**
+ * Gets the Ignite instance.
+ *
+ * @return Ignite instance.
+ */
+ public Ignite getIgnite() {
+ return ignite;
+ }
+
+ /**
* Obtains data flush frequency.
*
* @return Flush frequency.
@@ -109,12 +126,10 @@ public class IgniteSink<IN> extends RichSinkFunction<IN> {
* Default IgniteSink constructor.
*
* @param cacheName Cache name.
- * @param igniteCfgFile Ignite configuration file.
*/
public IgniteSink(String cacheName, String igniteCfgFile) {
this.cacheName = cacheName;
this.igniteCfgFile = igniteCfgFile;
- this.log = SinkContext.getIgnite().log();
}
/**
@@ -122,13 +137,26 @@ public class IgniteSink<IN> extends RichSinkFunction<IN> {
*
* @throws IgniteException If failed.
*/
+ @Override
@SuppressWarnings("unchecked")
- public void start() throws IgniteException {
+ public void open(Configuration parameter) {
A.notNull(igniteCfgFile, "Ignite config file");
A.notNull(cacheName, "Cache name");
- SinkContext.getStreamer().autoFlushFrequency(autoFlushFrequency);
- SinkContext.getStreamer().allowOverwrite(allowOverwrite);
+ try {
+ // if an ignite instance is already started in same JVM then use it.
+ this.ignite = Ignition.ignite();
+ } catch (IgniteIllegalStateException e) {
+ this.ignite = Ignition.start(igniteCfgFile);
+ }
+
+ this.ignite.getOrCreateCache(cacheName);
+
+ this.log = this.ignite.log();
+
+ this.streamer = this.ignite.dataStreamer(cacheName);
+ this.streamer.autoFlushFrequency(autoFlushFrequency);
+ this.streamer.allowOverwrite(allowOverwrite);
stopped = false;
}
@@ -138,15 +166,14 @@ public class IgniteSink<IN> extends RichSinkFunction<IN> {
*
* @throws IgniteException If failed.
*/
- public void stop() throws IgniteException {
+ @Override
+ public void close() {
if (stopped)
return;
stopped = true;
- SinkContext.getStreamer().close();
- SinkContext.getIgnite().cache(cacheName).close();
- SinkContext.getIgnite().close();
+ this.streamer.close();
}
/**
@@ -162,43 +189,10 @@ public class IgniteSink<IN> extends RichSinkFunction<IN> {
if (!(in instanceof Map))
throw new IgniteException("Map as a streamer input is expected!");
- SinkContext.getStreamer().addData((Map)in);
+ this.streamer.addData((Map)in);
}
catch (Exception e) {
log.error("Error while processing IN of " + cacheName, e);
}
}
-
- /**
- * Streamer context initializing grid and data streamer instances on demand.
- */
- private static class SinkContext {
- /** Constructor. */
- private SinkContext() {
- }
-
- /** Instance holder. */
- private static class Holder {
- private static final Ignite IGNITE = Ignition.start(igniteCfgFile);
- private static final IgniteDataStreamer STREAMER = IGNITE.dataStreamer(cacheName);
- }
-
- /**
- * Obtains grid instance.
- *
- * @return Grid instance.
- */
- private static Ignite getIgnite() {
- return Holder.IGNITE;
- }
-
- /**
- * Obtains data streamer instance.
- *
- * @return Data streamer instance.
- */
- private static IgniteDataStreamer getStreamer() {
- return Holder.STREAMER;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/891da2a5/modules/flink/src/test/java/org/apache/ignite/sink/flink/FlinkIgniteSinkSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/flink/src/test/java/org/apache/ignite/sink/flink/FlinkIgniteSinkSelfTest.java b/modules/flink/src/test/java/org/apache/ignite/sink/flink/FlinkIgniteSinkSelfTest.java
index 50eedb8..eb59379 100644
--- a/modules/flink/src/test/java/org/apache/ignite/sink/flink/FlinkIgniteSinkSelfTest.java
+++ b/modules/flink/src/test/java/org/apache/ignite/sink/flink/FlinkIgniteSinkSelfTest.java
@@ -19,19 +19,11 @@ package org.apache.ignite.sink.flink;
import java.util.HashMap;
import java.util.Map;
-import java.util.concurrent.CountDownLatch;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.events.CacheEvent;
-import org.apache.ignite.events.EventType;
-import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_PUT;
-
/**
* Tests for {@link IgniteSink}.
*/
@@ -39,147 +31,51 @@ public class FlinkIgniteSinkSelfTest extends GridCommonAbstractTest {
/** Cache name. */
private static final String TEST_CACHE = "testCache";
- /** Cache entries count. */
- private static final int CACHE_ENTRY_COUNT = 10000;
-
- /** Streaming events for testing. */
- private static final long DFLT_STREAMING_EVENT = 10000;
-
- /** Ignite instance. */
- private Ignite ignite;
-
/** Ignite test configuration file. */
private static final String GRID_CONF_FILE = "modules/flink/src/test/resources/example-ignite.xml";
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return 20_000;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override protected void beforeTest() throws Exception {
- IgniteConfiguration cfg = loadConfiguration(GRID_CONF_FILE);
-
- cfg.setClientMode(false);
-
- ignite = startGrid("igniteServerNode", cfg);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
- }
-
- /**
- * Tests for the Flink sink.
- * Ignite started in sink based on what is specified in the configuration file.
- *
- * @throws Exception
- */
- @SuppressWarnings("unchecked")
- public void testFlinkIgniteSink() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- env.getConfig().disableSysoutLogging();
+ public void testIgniteSink() throws Exception {
+ Configuration configuration = new Configuration();
IgniteSink igniteSink = new IgniteSink(TEST_CACHE, GRID_CONF_FILE);
igniteSink.setAllowOverwrite(true);
- igniteSink.setAutoFlushFrequency(10);
+ igniteSink.setAutoFlushFrequency(1L);
+
+ igniteSink.open(configuration);
- igniteSink.start();
+ Map<String, String> myData = new HashMap<>();
+ myData.put("testData", "testValue");
- CacheListener listener = subscribeToPutEvents();
+ igniteSink.invoke(myData);
- DataStream<Map> stream = env.addSource(new SourceFunction<Map>() {
+ /** waiting for a small duration for the cache flush to complete */
+ Thread.sleep(2000);
- private boolean running = true;
+ assertEquals("testValue", igniteSink.getIgnite().getOrCreateCache(TEST_CACHE).get("testData"));
+ }
- @Override public void run(SourceContext<Map> ctx) throws Exception {
- Map testDataMap = new HashMap<>();
- long cnt = 0;
+ public void testIgniteSinkStreamExecution() throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- while (running && (cnt < DFLT_STREAMING_EVENT)) {
- testDataMap.put(cnt, "ignite-" + cnt);
- cnt++;
- }
+ IgniteSink igniteSink = new IgniteSink(TEST_CACHE, GRID_CONF_FILE);
- ctx.collect(testDataMap);
- }
+ igniteSink.setAllowOverwrite(true);
- @Override public void cancel() {
- running = false;
- }
- }).setParallelism(1);
+ igniteSink.setAutoFlushFrequency(1);
- assertEquals(0, ignite.cache(TEST_CACHE).size());
+ Map<String, String> myData = new HashMap<>();
+ myData.put("testdata", "testValue");
+ DataStream<Map> stream = env.fromElements(myData);
- // sink data into the grid.
stream.addSink(igniteSink);
-
try {
env.execute();
-
- CountDownLatch latch = listener.getLatch();
-
- // Enough events was handled in 10 seconds. Limited by test's timeout.
- latch.await();
-
- unsubscribeToPutEvents(listener);
-
- assertEquals(DFLT_STREAMING_EVENT, ignite.getOrCreateCache(TEST_CACHE).size());
-
- for (long i = 0; i < DFLT_STREAMING_EVENT; i++)
- assertEquals("ignite-" + i, ignite.getOrCreateCache(TEST_CACHE).get(i));
-
- }
- finally {
- igniteSink.stop();
}
- }
-
- /**
- * Sets a listener for {@link EventType#EVT_CACHE_OBJECT_PUT}.
- *
- * @return Cache listener.
- */
- private CacheListener subscribeToPutEvents() {
- // Listen to cache PUT events and expect as many as messages as test data items.
- CacheListener listener = new CacheListener();
-
- ignite.events(ignite.cluster().forCacheNodes(DEFAULT_CACHE_NAME)).localListen(listener, EVT_CACHE_OBJECT_PUT);
-
- return listener;
- }
-
- /**
- * Removes the listener for {@link EventType#EVT_CACHE_OBJECT_PUT}.
- *
- * @param listener Cache listener.
- */
- private void unsubscribeToPutEvents(CacheListener listener) {
- ignite.events(ignite.cluster().forCacheNodes(DEFAULT_CACHE_NAME)).stopLocalListen(listener, EVT_CACHE_OBJECT_PUT);
- }
-
- /** Listener. */
- private class CacheListener implements IgnitePredicate<CacheEvent> {
- private final CountDownLatch latch = new CountDownLatch(CACHE_ENTRY_COUNT);
-
- /** @return Latch. */
- public CountDownLatch getLatch() {
- return latch;
- }
-
- /**
- * @param evt Cache Event.
- * @return {@code true}.
- */
- @Override public boolean apply(CacheEvent evt) {
- latch.countDown();
-
- return true;
+ catch (Exception e) {
+ e.printStackTrace();
+ fail("Stream execution process failed.");
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/891da2a5/modules/flink/src/test/resources/example-ignite.xml
----------------------------------------------------------------------
diff --git a/modules/flink/src/test/resources/example-ignite.xml b/modules/flink/src/test/resources/example-ignite.xml
index b8ddc8f..d4f4dc1 100644
--- a/modules/flink/src/test/resources/example-ignite.xml
+++ b/modules/flink/src/test/resources/example-ignite.xml
@@ -31,7 +31,7 @@
http://www.springframework.org/schema/util/spring-util.xsd">
<bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
<!-- Enable client mode. -->
- <property name="clientMode" value="true"/>
+ <property name="clientMode" value="false"/>
<!-- Cache accessed from IgniteSink. -->
<property name="cacheConfiguration">
@@ -49,6 +49,9 @@
<list>
<!-- Cache events (only EVT_CACHE_OBJECT_PUT for tests). -->
<util:constant static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_PUT"/>
+ <util:constant static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_READ"/>
+ <util:constant static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_REMOVED"/>
+
</list>
</property>
@@ -59,7 +62,7 @@
<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
<property name="addresses">
<list>
- <value>127.0.0.1:47500</value>
+ <value>127.0.0.1:47500..47509</value>
</list>
</property>
</bean>
[24/50] [abbrv] ignite git commit: IGNITE-9137 Web Console: Select
node dialog refactored to new UI.
Posted by av...@apache.org.
IGNITE-9137 Web Console: Select node dialog refactored to new UI.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8ea35e74
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8ea35e74
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8ea35e74
Branch: refs/heads/ignite-8446
Commit: 8ea35e74e1067babfbc68249f1565f18433b2fb9
Parents: ccd65e6
Author: Dmitriy Shabalin <dm...@gmail.com>
Authored: Wed Aug 1 11:08:54 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Aug 1 11:08:54 2018 +0700
----------------------------------------------------------------------
.../modules/nodes/nodes-dialog.controller.js | 6 ++---
.../app/modules/nodes/nodes-dialog.scss | 17 +++++-------
.../app/modules/nodes/nodes-dialog.tpl.pug | 28 ++++++++++++--------
3 files changed, 26 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8ea35e74/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js
index fbe6203..2296aed 100644
--- a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js
+++ b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js
@@ -18,9 +18,9 @@
const NID_TEMPLATE = '<div class="ui-grid-cell-contents" title="{{ COL_FIELD }}">{{ COL_FIELD | limitTo:8 }}</div>';
const COLUMNS_DEFS = [
- {displayName: 'Node ID8', field: 'nid', headerTooltip: 'Node ID8', cellTemplate: NID_TEMPLATE, minWidth: 85, width: 85, pinnedLeft: true},
- {displayName: 'Node IP', field: 'ip', headerTooltip: 'Primary IP address of node', minWidth: 75, width: 120},
- {displayName: 'Grid name', field: 'gridName', headerTooltip: 'Name of node grid cluster', minWidth: 75, width: 120},
+ {displayName: 'Node ID8', field: 'nid', headerTooltip: 'Node ID8', cellTemplate: NID_TEMPLATE, minWidth: 85, width: 145, pinnedLeft: true},
+ {displayName: 'Node IP', field: 'ip', headerTooltip: 'Primary IP address of node', minWidth: 100, width: 150},
+ {displayName: 'Grid name', field: 'gridName', headerTooltip: 'Name of node grid cluster', minWidth: 110, width: 150},
{displayName: 'Version', field: 'version', headerTooltip: 'Node version', minWidth: 75, width: 140},
{displayName: 'OS information', field: 'os', headerTooltip: 'OS information for node\'s host', minWidth: 125}
];
http://git-wip-us.apache.org/repos/asf/ignite/blob/8ea35e74/modules/web-console/frontend/app/modules/nodes/nodes-dialog.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.scss b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.scss
index 8145dbc..cbb9000 100644
--- a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.scss
+++ b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.scss
@@ -16,22 +16,17 @@
*/
.ignite-nodes-dialog {
- label {
- font-size: 18px;
- margin-right: 20px;
- }
-
.ui-grid-pinned-container.ui-grid-pinned-container-left .ui-grid-cell:last-child,
.ui-grid-pinned-container.ui-grid-pinned-container-left .ui-grid-header-cell:last-child,
.ui-grid-header-cell:last-child .ui-grid-column-resizer.right {
- //border-right: none;
+ border-right: none;
}
- .nodes-grid {
- height: 320px;
+ .modal-dialog {
+ width: 900px;
}
- .panel-body_collapse {
- padding: 0;
- margin: 0;
+
+ button.pull-left.btn-ignite {
+ background: none;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8ea35e74/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
index 21b138b..22f3b1d 100644
--- a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
+++ b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
@@ -14,22 +14,28 @@
See the License for the specific language governing permissions and
limitations under the License.
-.modal.ignite-nodes-dialog(tabindex='-1' role='dialog')
- .modal-dialog
- .modal-content
+.modal.modal--ignite.theme--ignite.ignite-nodes-dialog(tabindex='-1' role='dialog')
+ .modal-dialog.modal-dialog--adjust-height
+ form.modal-content
.modal-header
- button.close(ng-click='$cancel()' aria-hidden='true') ×
h4.modal-title Select Node
+ button.close(type='button' aria-label='Close' ng-click='$cancel()')
+ svg(ignite-icon="cross")
.modal-body.modal-body-with-scroll
p Choose node to execute query for cache: #[strong {{ $ctrl.options.target }}]
- .panel.panel-default.nodes-grid
- .panel-heading
- label Cache Nodes: {{ $ctrl.nodes.length }}
+ ul.tabs.tabs--blue
+ li.active(role='presentation')
+ a
+ span Cache Nodes
+ span.badge.badge--blue {{ $ctrl.data.length }}
- .panel-body.panel-body_collapse
- .grid(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-pinning ui-grid-hovering)
+ .panel--ignite
+ .grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-pinning ui-grid-hovering)
.modal-footer
- button.btn.btn-primary(id='confirm-btn-confirm' ng-click='$ok($ctrl.selected)' ng-disabled='$ctrl.selected.length === 0') Select node
- button.btn.btn-default(id='confirm-btn-close' ng-click='$cancel()') Cancel
+ button.pull-left.btn-ignite(disabled)
+ grid-item-selected(class='pull-left' grid-api='$ctrl.gridApi')
+
+ button.btn-ignite.btn-ignite--link-success(id='confirm-btn-close' ng-click='$cancel()') Cancel
+ button.btn-ignite.btn-ignite--success(id='confirm-btn-confirm' ng-click='$ok($ctrl.selected)' ng-disabled='$ctrl.selected.length === 0') Select node
[19/50] [abbrv] ignite git commit: IGNITE-9034: [ML] Add Estimator
API support to TensorFlow cluster on top of Apache Ignite.
Posted by av...@apache.org.
IGNITE-9034: [ML] Add Estimator API support to TensorFlow
cluster on top of Apache Ignite.
this closes #4402
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9e884e5a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9e884e5a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9e884e5a
Branch: refs/heads/ignite-8446
Commit: 9e884e5aaa3fba3a68967c51280f3a8f41514f51
Parents: c5e723c
Author: Anton Dmitriev <dm...@gmail.com>
Authored: Tue Jul 31 16:54:26 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Tue Jul 31 16:54:26 2018 +0300
----------------------------------------------------------------------
examples/pom.xml | 21 --
.../cluster/TensorFlowClusterExample.java | 118 --------
modules/tensorflow/README.txt | 55 +++-
modules/tensorflow/pom.xml | 56 +++-
modules/tensorflow/src/main/assembly/zip.xml | 56 ++++
.../cluster/TensorFlowClusterGateway.java | 34 ++-
.../TensorFlowClusterGatewayManager.java | 95 +++++--
.../cluster/TensorFlowClusterMaintainer.java | 105 +++++---
.../cluster/TensorFlowClusterManager.java | 268 +++++++++++++------
.../cluster/TensorFlowJobArchive.java | 65 +++++
.../cluster/spec/TensorFlowClusterSpec.java | 39 +++
.../spec/TensorFlowServerAddressSpec.java | 14 +
.../tfrunning/TensorFlowServerManager.java | 112 +-------
.../TensorFlowServerScriptFormatter.java | 62 +++++
.../cluster/util/ClusterPortManager.java | 168 +++++++++---
.../cluster/util/TensorFlowChiefRunner.java | 80 ++++++
.../cluster/util/TensorFlowClusterResolver.java | 92 ++++---
.../util/TensorFlowUserScriptRunner.java | 236 ++++++++++++++++
.../ignite/tensorflow/core/ProcessManager.java | 5 +-
.../tensorflow/core/ProcessManagerWrapper.java | 5 +-
.../longrunning/LongRunningProcessManager.java | 19 +-
.../task/LongRunningProcessStartTask.java | 4 +-
.../core/nativerunning/NativeProcess.java | 6 +-
.../nativerunning/NativeProcessManager.java | 12 +-
.../task/NativeProcessStartTask.java | 81 ++----
.../PythonProcessBuilderSupplier.java | 57 ++++
.../pythonrunning/PythonProcessManager.java | 35 +--
.../core/util/AsyncNativeProcessRunner.java | 107 ++++++++
.../core/util/NativeProcessRunner.java | 133 +++++++++
.../tensorflow/submitter/JobSubmitter.java | 35 +++
.../submitter/command/AbstractCommand.java | 55 ++++
.../submitter/command/AttachCommand.java | 51 ++++
.../tensorflow/submitter/command/PsCommand.java | 47 ++++
.../submitter/command/RootCommand.java | 42 +++
.../submitter/command/StartCommand.java | 205 ++++++++++++++
.../submitter/command/StopCommand.java | 50 ++++
.../submitter/command/package-info.java | 22 ++
.../tensorflow/submitter/package-info.java | 23 ++
.../tensorflow/util/SerializableConsumer.java | 29 ++
.../tensorflow/util/SerializableSupplier.java | 29 ++
.../ignite/tensorflow/util/package-info.java | 22 ++
modules/tensorflow/src/main/sh/ignite-tf.sh | 19 ++
modules/tensorflow/src/main/sh/logback.xml | 36 +++
.../LongRunningProcessManagerTest.java | 10 +-
44 files changed, 2222 insertions(+), 593 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index 3f6e6a8..e745beb 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -124,8 +124,6 @@
<lgpl.test.folder>src/test/java</lgpl.test.folder>
<spark.folder>src/main/java</spark.folder>
<spark.test.folder>src/test/java</spark.test.folder>
- <tensorflow.folder>src/main/java</tensorflow.folder>
- <tensorflow.test.folder>src/test/java</tensorflow.test.folder>
</properties>
<profiles>
@@ -202,23 +200,6 @@
</dependency>
</dependencies>
</profile>
-
- <profile>
- <id>tensorflow</id>
-
- <properties>
- <tensorflow.folder>src/main/tensorflow</tensorflow.folder>
- <tensorflow.test.folder>src/test/tensorflow</tensorflow.test.folder>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-tensorflow</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
</profiles>
<build>
@@ -249,7 +230,6 @@
<sources>
<source>${lgpl.folder}</source>
<source>${spark.folder}</source>
- <source>${tensorflow.folder}</source>
</sources>
</configuration>
</execution>
@@ -264,7 +244,6 @@
<sources>
<source>${lgpl.test.folder}</source>
<source>${spark.test.folder}</source>
- <source>${tensorflow.test.folder}</source>
</sources>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/examples/src/main/tensorflow/org/apache/ignite/tensorflow/cluster/TensorFlowClusterExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/tensorflow/org/apache/ignite/tensorflow/cluster/TensorFlowClusterExample.java b/examples/src/main/tensorflow/org/apache/ignite/tensorflow/cluster/TensorFlowClusterExample.java
deleted file mode 100644
index 3a956c9..0000000
--- a/examples/src/main/tensorflow/org/apache/ignite/tensorflow/cluster/TensorFlowClusterExample.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tensorflow.cluster;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
-import java.util.function.Supplier;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.tensorflow.cluster.tfrunning.TensorFlowServerManager;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-
-/**
- * Prerequisites: be aware that to successfully run this example you need to have Python and TensorFlow installed on
- * your machine. To find out how to install Python please take a look this page: https://www.python.org/downloads/. To
- * install TensorFlow see this web site: https://www.tensorflow.org/install/.
- *
- * Example that shows how to use {@link TensorFlowClusterGatewayManager} and start, maintain and stop TensorFlow
- * cluster.
- */
-public class TensorFlowClusterExample {
- /** Run example. */
- public static void main(String... args) throws InterruptedException {
- IgniteConfiguration configuration = new IgniteConfiguration();
- configuration.setClientMode(false);
-
- try (Ignite ignite = Ignition.start(configuration)) {
- System.out.println(">>> TensorFlow cluster example started.");
-
- CacheConfiguration<Integer, Integer> cacheConfiguration = new CacheConfiguration<>();
- cacheConfiguration.setAffinity(new RendezvousAffinityFunction(false, 10));
- cacheConfiguration.setName("TEST_CACHE");
-
- IgniteCache<Integer, Integer> cache = ignite.getOrCreateCache(cacheConfiguration);
- for (int i = 0; i < 1000; i++)
- cache.put(i, i);
-
- System.out.println(">>> Cache created.");
-
- TensorFlowClusterGatewayManager mgr = new TensorFlowClusterGatewayManager(ignite);
- TensorFlowClusterGateway gateway = mgr.getOrCreateCluster("TEST_CACHE");
-
- System.out.println(">>> TensorFlow cluster gateway started.");
-
- CountDownLatch latch = new CountDownLatch(1);
-
- gateway.subscribe(cluster -> {
- StringBuilder builder = new StringBuilder();
- builder.append("------------------- TensorFlow Cluster Service Info -------------------").append('\n');
-
- builder.append("Specification : ").append('\n');
-
- TensorFlowServerManager srvMgr = new TensorFlowServerManager(
- (Supplier<Ignite> & Serializable)() -> ignite
- );
-
- String clusterSpec = srvMgr.formatClusterSpec(cluster.getSpec());
- builder.append(clusterSpec).append('\n');
-
- Map<UUID, List<LongRunningProcessStatus>> statuses = srvMgr.ping(cluster.getProcesses());
-
- builder.append("State : ").append('\n');
-
- for (UUID nodeId : cluster.getProcesses().keySet()) {
- List<UUID> pr = cluster.getProcesses().get(nodeId);
- List<LongRunningProcessStatus> st = statuses.get(nodeId);
-
- builder.append("Node ").append(nodeId.toString().substring(0, 8)).append(" -> ").append('\n');
- for (int i = 0; i < pr.size(); i++) {
- builder.append("\tProcess ")
- .append(pr.get(i).toString().substring(0, 8))
- .append(" with status ")
- .append(st.get(i).getState());
-
- if (st.get(i).getException() != null)
- builder.append(" (").append(st.get(i).getException()).append(")");
-
- builder.append('\n');
- }
- }
-
- builder.append("-----------------------------------------------------------------------").append('\n');
-
- System.out.println(builder);
-
- latch.countDown();
- });
-
- latch.await();
-
- mgr.stopClusterIfExists("TEST_CACHE");
-
- System.out.println(">>> TensorFlow cluster example completed.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/README.txt
----------------------------------------------------------------------
diff --git a/modules/tensorflow/README.txt b/modules/tensorflow/README.txt
index 45fc3cf..21ea88d 100644
--- a/modules/tensorflow/README.txt
+++ b/modules/tensorflow/README.txt
@@ -7,7 +7,7 @@ will be a datasource for any TensorFlow model training.
Import Apache Ignite TensorFlow Integration Module In Maven Project
-------------------------------------
-If you are using Maven to manage dependencies of your project, you can add Cassandra Store module
+If you are using Maven to manage dependencies of your project, you can add TensorFlow module
dependency like this (replace '${ignite.version}' with actual Ignite version you are
interested in):
@@ -26,4 +26,55 @@ interested in):
...
</dependencies>
...
-</project>
\ No newline at end of file
+</project>
+-------------------------------------
+
+TensorFlow integration module provides command line tool that allows to start, maintain and stop distributed deep
+learning utilizing Apache Ignite infrastructure and data. This tool provides several commands that are shown here:
+
+Usage: ignite-tf [-hV] [-c=<cfg>] [COMMAND]
+Apache Ignite and TensorFlow integration command line utility that allows to
+start, maintain and stop distributed deep learning utilizing Apache Ignite
+infrastructure and data.
+ -c, --config=<cfg> Apache Ignite client configuration.
+ -h, --help Show this help message and exit.
+ -V, --version Print version information and exit.
+Commands:
+ start Starts a new TensorFlow cluster and attaches to user script process.
+ stop Stops a running TensorFlow cluster.
+ attach Attaches to running TensorFlow cluster (user script process).
+ ps Prints identifiers of all running TensorFlow clusters.
+
+To start TensorFlow cluster you need to specify upstream cache that will be used as data source for training, folder
+that contains code that actually performs training and command that should be called on this code to start training
+correctly. Command "start" have the following help output:
+
+Usage: ignite-tf start [-hV] [-c=<cfg>] CACHE_NAME JOB_DIR JOB_CMD [JOB_ARGS...]
+Starts a new TensorFlow cluster and attaches to user script process.
+ CACHE_NAME Upstream cache name.
+ JOB_DIR Job folder (or zip archive).
+ JOB_CMD Job command.
+ [JOB_ARGS...] Job arguments.
+ -c, --config=<cfg> Apache Ignite client configuration.
+ -h, --help Show this help message and exit.
+ -V, --version Print version information and exit.
+
+To attach to running TensorFlow cluster or stop it you can use commands "attach" and "stop" correspondingly. These
+commands accepts cluster identifier as a parameter:
+
+Usage: ignite-tf attach [-hV] [-c=<cfg>] CLUSTER_ID
+Attaches to running TensorFlow cluster (user script process).
+ CLUSTER_ID Cluster identifier.
+ -c, --config=<cfg> Apache Ignite client configuration.
+ -h, --help Show this help message and exit.
+ -V, --version Print version information and exit.
+
+Usage: ignite-tf stop [-hV] [-c=<cfg>] CLUSTER_ID
+Stops a running TensorFlow cluster.
+ CLUSTER_ID Cluster identifier.
+ -c, --config=<cfg> Apache Ignite client configuration.
+ -h, --help Show this help message and exit.
+ -V, --version Print version information and exit.
+
+To find out what TensorFlow clusters are currently running on top of Apache Ignite you can use "ps" command that doesn't
+require arguments.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/pom.xml
----------------------------------------------------------------------
diff --git a/modules/tensorflow/pom.xml b/modules/tensorflow/pom.xml
index 72e7e69..b1cfe21 100644
--- a/modules/tensorflow/pom.xml
+++ b/modules/tensorflow/pom.xml
@@ -34,6 +34,30 @@
<version>2.7.0-SNAPSHOT</version>
<url>http://ignite.apache.org</url>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/main/assembly/zip.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
<dependencies>
<dependency>
<groupId>org.apache.ignite</groupId>
@@ -43,22 +67,39 @@
<dependency>
<groupId>org.apache.ignite</groupId>
- <artifactId>ignite-core</artifactId>
+ <artifactId>ignite-spring</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.ignite</groupId>
- <artifactId>ignite-spring</artifactId>
+ <artifactId>ignite-slf4j</artifactId>
<version>${project.version}</version>
- <scope>test</scope>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>1.2.3</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.6</version>
+ </dependency>
+
+ <dependency>
+ <groupId>info.picocli</groupId>
+ <artifactId>picocli</artifactId>
+ <version>3.3.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -86,5 +127,4 @@
</build>
</profile>
</profiles>
-
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/assembly/zip.xml
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/assembly/zip.xml b/modules/tensorflow/src/main/assembly/zip.xml
new file mode 100644
index 0000000..0c90e98
--- /dev/null
+++ b/modules/tensorflow/src/main/assembly/zip.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ Assembly file.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
+ http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>zip</id>
+ <includeBaseDirectory>true</includeBaseDirectory>
+
+ <formats>
+ <format>zip</format>
+ </formats>
+
+ <files>
+ <file>
+ <source>${project.basedir}/src/main/sh/ignite-tf.sh</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+ <file>
+ <source>${project.basedir}/src/main/sh/logback.xml</source>
+ <outputDirectory>/</outputDirectory>
+ </file>
+ <file>
+ <source>${project.build.directory}/${project.artifactId}-${project.version}.jar</source>
+ <outputDirectory>/lib</outputDirectory>
+ </file>
+ </files>
+
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ <useStrictFiltering>true</useStrictFiltering>
+ <useProjectArtifact>false</useProjectArtifact>
+ <scope>runtime</scope>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGateway.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGateway.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGateway.java
index 5eee155..092dfcb 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGateway.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGateway.java
@@ -18,29 +18,43 @@
package org.apache.ignite.tensorflow.cluster;
import java.util.HashSet;
+import java.util.Optional;
import java.util.UUID;
import java.util.function.Consumer;
import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.tensorflow.util.SerializableConsumer;
/**
* TensorFlow cluster gateway that allows to subscribe on changes in cluster configuration.
*/
-public class TensorFlowClusterGateway implements IgniteBiPredicate<UUID, TensorFlowCluster> {
+public class TensorFlowClusterGateway implements IgniteBiPredicate<UUID, Optional<TensorFlowCluster>>, AutoCloseable {
/** */
private static final long serialVersionUID = -540323262800791340L;
+ /** Callback that will be called on unsubscribe. */
+ private final SerializableConsumer<TensorFlowClusterGateway> unsubscribeCb;
+
/** Subscribers. */
- private final HashSet<Consumer<TensorFlowCluster>> subscribers = new HashSet<>();
+ private final HashSet<Consumer<Optional<TensorFlowCluster>>> subscribers = new HashSet<>();
/** Last value received from the upstream. */
- private TensorFlowCluster last;
+ private Optional<TensorFlowCluster> last;
+
+ /**
+ * Constructs a new instance of TensorFlow cluster gateway.
+ *
+ * @param unsubscribeCb Callback that will be called on unsubscribe.
+ */
+ public TensorFlowClusterGateway(SerializableConsumer<TensorFlowClusterGateway> unsubscribeCb) {
+ this.unsubscribeCb = unsubscribeCb;
+ }
/**
* Subscribers the specified subscriber on the upstream events.
*
* @param subscriber Subscriber.
*/
- public synchronized void subscribe(Consumer<TensorFlowCluster> subscriber) {
+ public synchronized void subscribe(Consumer<Optional<TensorFlowCluster>> subscriber) {
subscribers.add(subscriber);
if (last != null)
@@ -52,17 +66,23 @@ public class TensorFlowClusterGateway implements IgniteBiPredicate<UUID, TensorF
*
* @param subscriber Subscriber.
*/
- public synchronized void unsubscribe(Consumer<TensorFlowCluster> subscriber) {
+ public synchronized void unsubscribe(Consumer<Optional<TensorFlowCluster>> subscriber) {
subscribers.remove(subscriber);
}
/** {@inheritDoc} */
- @Override public synchronized boolean apply(UUID uuid, TensorFlowCluster cluster) {
- for (Consumer<TensorFlowCluster> subscriber : subscribers)
+ @Override public synchronized boolean apply(UUID uuid, Optional<TensorFlowCluster> cluster) {
+ for (Consumer<Optional<TensorFlowCluster>> subscriber : subscribers)
subscriber.accept(cluster);
last = cluster;
return true;
}
+
+ /** {@inheritDoc} */
+ @Override public void close() {
+ subscribers.clear();
+ unsubscribeCb.accept(this);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGatewayManager.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGatewayManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGatewayManager.java
index f4b8187..a315aaf 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGatewayManager.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterGatewayManager.java
@@ -17,8 +17,12 @@
package org.apache.ignite.tensorflow.cluster;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.function.Consumer;
import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteServices;
+import org.apache.ignite.IgniteLogger;
/**
* TensorFlow cluster manager that allows to start, maintain and stop TensorFlow cluster using
@@ -34,6 +38,9 @@ public class TensorFlowClusterGatewayManager {
/** Ignite instance. */
private final Ignite ignite;
+ /** Logger. */
+ private final IgniteLogger log;
+
/**
* Constructs a new instance of TensorFlow cluster manager with maintenance.
*
@@ -43,37 +50,89 @@ public class TensorFlowClusterGatewayManager {
assert ignite != null : "Ignite should not be null";
this.ignite = ignite;
+ this.log = ignite.log().getLogger(TensorFlowClusterGatewayManager.class);
}
/**
- * Creates and starts a new TensorFlow cluster for the specified cache if it doesn't exist, otherwise returns
- * existing one.
+ * Subscribes on changes of the specified cluster.
*
- * @param upstreamCacheName Upstream cache name.
+ * @param clusterId Cluster identifier.
* @return TensorFlow cluster gateway that allows to subscribe on cluster changes.
*/
- public TensorFlowClusterGateway getOrCreateCluster(String upstreamCacheName) {
- String svcName = String.format(SERVICE_NAME_TEMPLATE, upstreamCacheName);
- String topicName = String.format(SERVICE_TOPIC_NAME_TEMPLATE, upstreamCacheName);
+ public TensorFlowClusterGateway getCluster(UUID clusterId) {
+ String topicName = String.format(SERVICE_TOPIC_NAME_TEMPLATE, clusterId);
- TensorFlowClusterGateway gateway = createTensorFlowClusterGateway(topicName);
+ return createTensorFlowClusterGateway(topicName);
+ }
- IgniteServices services = ignite.services();
+ /**
+ * Creates and starts a new TensorFlow cluster for the specified cache.
+ *
+ * @param clusterId Cluster identifier.
+ * @param jobArchive Job archive.
+ * @return TensorFlow cluster gateway that allows to subscribe on cluster changes.
+ */
+ public TensorFlowClusterGateway createCluster(UUID clusterId, TensorFlowJobArchive jobArchive) {
+ String svcName = String.format(SERVICE_NAME_TEMPLATE, clusterId);
+ String topicName = String.format(SERVICE_TOPIC_NAME_TEMPLATE, clusterId);
- services.deployClusterSingleton(svcName, new TensorFlowClusterMaintainer(upstreamCacheName, topicName));
+ TensorFlowClusterGateway gateway = createTensorFlowClusterGateway(topicName);
+
+ ignite.services().deployClusterSingleton(
+ svcName,
+ new TensorFlowClusterMaintainer(clusterId, jobArchive, topicName)
+ );
+ log.info("Cluster maintainer deployed as a service [clusterId=" + clusterId + "]");
return gateway;
}
/**
- * Stops TensorFlow cluster.
+ * Listens to TensorFlow cluster user script.
*
- * @param upstreamCacheName Upstream cache name.
+ * @param clusterId Cluster identifier.
+ * @param out Output stream consumer.
+ * @param err Error stream consumer.
*/
- public void stopClusterIfExists(String upstreamCacheName) {
- IgniteServices services = ignite.services();
+ public void listenToClusterUserScript(UUID clusterId, Consumer<String> out, Consumer<String> err) {
+ TensorFlowClusterGateway gateway = getCluster(clusterId);
+
+ ignite.message().localListen("us_out_" + clusterId, (node, msg) -> {
+ out.accept(msg.toString());
+ return true;
+ });
+
+ ignite.message().localListen("us_err_" + clusterId, (node, msg) -> {
+ err.accept(msg.toString());
+ return true;
+ });
+
+ CountDownLatch latch = new CountDownLatch(1);
+
+ Consumer<Optional<TensorFlowCluster>> subscriber = cluster -> {
+ if (!cluster.isPresent())
+ latch.countDown();
+ };
+
+ gateway.subscribe(subscriber);
+
+ try {
+ latch.await();
+ gateway.unsubscribe(subscriber);
+ }
+ catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
- services.cancel(String.format(SERVICE_NAME_TEMPLATE, upstreamCacheName));
+ /**
+ * Stops TensorFlow cluster.
+ *
+ * @param clusterId Cluster identifier.
+ */
+ public void stopClusterIfExists(UUID clusterId) {
+ ignite.services().cancel(String.format(SERVICE_NAME_TEMPLATE, clusterId));
+ log.info("Cluster maintained cancelled as a service [clusterId=" + clusterId + "]");
}
/**
@@ -83,9 +142,13 @@ public class TensorFlowClusterGatewayManager {
* @return TensorFlow cluster gateway.
*/
private TensorFlowClusterGateway createTensorFlowClusterGateway(String topicName) {
- TensorFlowClusterGateway gateway = new TensorFlowClusterGateway();
+ TensorFlowClusterGateway gateway = new TensorFlowClusterGateway(subscriber -> {
+ ignite.message().stopLocalListen(topicName, subscriber);
+ log.info("Stop listen to cluster gateway [topicName=" + topicName + "]");
+ });
ignite.message().localListen(topicName, gateway);
+ log.info("Start listen to cluster gateway [topicName=" + topicName + "]");
return gateway;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterMaintainer.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterMaintainer.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterMaintainer.java
index 21b9c2b..e6ca33d 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterMaintainer.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterMaintainer.java
@@ -17,17 +17,18 @@
package org.apache.ignite.tensorflow.cluster;
-import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.UUID;
-import java.util.function.Supplier;
+import java.util.concurrent.locks.LockSupport;
import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteMessaging;
-import org.apache.ignite.Ignition;
+import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
import org.apache.ignite.services.Service;
import org.apache.ignite.services.ServiceContext;
import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessState;
@@ -40,74 +41,116 @@ public class TensorFlowClusterMaintainer implements Service {
/** */
private static final long serialVersionUID = -3220563310643566419L;
- /** Upstream cache name. */
- private final String cacheName;
+ /** Ignite instance. */
+ @IgniteInstanceResource
+ private transient Ignite ignite;
+
+ /** Logger. */
+ @LoggerResource
+ private transient IgniteLogger log;
+
+ /** TensorFlow cluster identifier. */
+ private final UUID clusterId;
+
+ /** Job archive. */
+ private final TensorFlowJobArchive jobArchive;
/** Topic name. */
private final String topicName;
/** TensorFlow cluster manager. */
- private final TensorFlowClusterManager clusterMgr;
+ private transient TensorFlowClusterManager clusterMgr;
/** Previous partition mapping. */
- private UUID[] prev;
+ private transient UUID[] prev;
/**
* Constructs a new instance of TensorFlow cluster service.
*
- * @param cacheName Upstream cache name.
+ * @param clusterId Cluster identifier.
+ * @param jobArchive Job archive.
* @param topicName Topic name.
*/
- public TensorFlowClusterMaintainer(String cacheName, String topicName) {
- assert cacheName != null : "Cache name should not be null";
+ public TensorFlowClusterMaintainer(UUID clusterId, TensorFlowJobArchive jobArchive, String topicName) {
+ assert clusterId != null : "Cluster identifier should not be null";
+ assert jobArchive != null : "Job archive should not be null";
assert topicName != null : "Topic name should not be null";
- this.clusterMgr = new TensorFlowClusterManager((Supplier<Ignite> & Serializable)Ignition::ignite);
- this.cacheName = cacheName;
+ this.clusterId = clusterId;
+ this.jobArchive = jobArchive;
this.topicName = topicName;
}
/** {@inheritDoc} */
@Override public void cancel(ServiceContext ctx) {
- clusterMgr.stopClusterIfExists(cacheName);
+ clusterMgr.stopClusterIfExists(clusterId);
+ log.debug("Cluster maintainer canceled [clusterId=" + clusterId + "]");
}
/** {@inheritDoc} */
@Override public void init(ServiceContext ctx) {
- clusterMgr.init();
+ clusterMgr = new TensorFlowClusterManager(ignite);
+ log.debug("Cluster maintainer initialized [clusterId=" + clusterId + "]");
}
/** {@inheritDoc} */
- @Override public void execute(ServiceContext ctx) throws Exception {
+ @Override public void execute(ServiceContext ctx) {
while (!ctx.isCancelled()) {
- Thread.sleep(1000);
+ LockSupport.parkNanos(1_000_000);
+
+ boolean completed = clusterMgr.isUserScriptCompleted(clusterId);
+ if (completed)
+ break;
boolean restartRequired = hasAffinityChanged();
+ if (restartRequired)
+ log.debug("Affinity mapping changed, cluster will be restarted [clusterId=" + clusterId + "]");
+
if (!restartRequired) {
- TensorFlowCluster cluster = clusterMgr.getCache().get(cacheName);
- Map<UUID, List<LongRunningProcessStatus>> statuses = clusterMgr.getSrvProcMgr()
- .ping(cluster.getProcesses());
-
- for (UUID nodeId : statuses.keySet()) {
- for (LongRunningProcessStatus status : statuses.get(nodeId)) {
- if (status.getState().equals(LongRunningProcessState.DONE)) {
- restartRequired = true;
- break;
+ try {
+ TensorFlowCluster cluster = clusterMgr.getCluster(clusterId);
+ Map<UUID, List<LongRunningProcessStatus>> statuses = clusterMgr.getSrvProcMgr()
+ .ping(cluster.getProcesses());
+
+ for (UUID nodeId : statuses.keySet()) {
+ for (LongRunningProcessStatus status : statuses.get(nodeId)) {
+ if (status.getState().equals(LongRunningProcessState.DONE)) {
+ restartRequired = true;
+ break;
+ }
}
}
+
}
+ catch (Exception e) {
+ log.error("Failed to check process statuses", e);
+ restartRequired = true;
+ }
+
+ if (restartRequired)
+ log.debug("Fail detected, cluster will be restarted [clusterId=" + clusterId + "]");
}
if (restartRequired) {
- clusterMgr.stopClusterIfExists(cacheName);
+ clusterMgr.stopClusterIfExists(clusterId);
- TensorFlowCluster cluster = clusterMgr.getOrCreateCluster(cacheName);
+ TensorFlowCluster cluster = clusterMgr.createCluster(
+ clusterId,
+ jobArchive,
+ str -> ignite.message().sendOrdered("us_out_" + clusterId, str, 60 * 1000),
+ str -> ignite.message().sendOrdered("us_err_" + clusterId, str, 60 * 1000)
+ );
- IgniteMessaging messaging = Ignition.ignite().message();
- messaging.send(topicName, cluster);
+ ignite.message().send(topicName, Optional.of(cluster));
}
}
+
+ clusterMgr.stopClusterIfExists(clusterId);
+
+ ignite.message().send(topicName, Optional.empty());
+
+ log.debug("Cluster maintainer completed [clusterId=" + clusterId + "]");
}
/**
@@ -116,7 +159,7 @@ public class TensorFlowClusterMaintainer implements Service {
* @return True if mapping has been changed, otherwise false.
*/
private boolean hasAffinityChanged() {
- Affinity<?> affinity = Ignition.ignite().affinity(cacheName);
+ Affinity<?> affinity = ignite.affinity(jobArchive.getUpstreamCacheName());
int parts = affinity.partitions();
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterManager.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterManager.java
index 2d63195..cdbd774 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterManager.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowClusterManager.java
@@ -17,36 +17,42 @@
package org.apache.ignite.tensorflow.cluster;
-import java.io.Serializable;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executors;
import java.util.concurrent.locks.Lock;
-import java.util.function.Supplier;
+import java.util.function.Consumer;
+import javax.cache.Cache;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.tensorflow.cluster.spec.TensorFlowClusterSpec;
import org.apache.ignite.tensorflow.cluster.spec.TensorFlowServerAddressSpec;
import org.apache.ignite.tensorflow.cluster.tfrunning.TensorFlowServer;
import org.apache.ignite.tensorflow.cluster.tfrunning.TensorFlowServerManager;
+import org.apache.ignite.tensorflow.cluster.util.TensorFlowChiefRunner;
import org.apache.ignite.tensorflow.cluster.util.TensorFlowClusterResolver;
+import org.apache.ignite.tensorflow.cluster.util.TensorFlowUserScriptRunner;
+import org.apache.ignite.tensorflow.core.util.CustomizableThreadFactory;
/**
* TensorFlow cluster manager that allows to start, maintain and stop TensorFlow cluster.
*/
-public class TensorFlowClusterManager implements Serializable {
- /** */
- private static final long serialVersionUID = -4847155592164802806L;
-
+public class TensorFlowClusterManager {
/** TensorFlow cluster metadata cache name. */
private static final String TF_CLUSTER_METADATA_CACHE_NAME = "TF_CLUSTER_METADATA_CACHE";
- /** Ignite instance supplier. */
- private final Supplier<Ignite> igniteSupplier;
+ /** Ignite instance. */
+ private final Ignite ignite;
/** TensorFlow server manager. */
private final TensorFlowServerManager srvProcMgr;
@@ -55,74 +61,61 @@ public class TensorFlowClusterManager implements Serializable {
private final TensorFlowClusterResolver clusterRslvr;
/** TensorFlow cluster metadata cache. */
- private transient IgniteCache<String, TensorFlowCluster> cache;
+ private IgniteCache<UUID, TensorFlowCluster> cache;
+
+ /** TensorFlow chief runners. */
+ private ConcurrentMap<UUID, TensorFlowChiefRunner> chiefRunners;
+
+ /** TensorFlow user script runners. */
+ private ConcurrentMap<UUID, TensorFlowUserScriptRunner> userScriptRunners;
/**
* Constructs a new instance of TensorFlow cluster manager.
*
- * @param igniteSupplier Ignite instance supplier.
- * @param <T> Type of serializable supplier.
+ * @param ignite Ignite instance.
*/
- public <T extends Supplier<Ignite> & Serializable> TensorFlowClusterManager(T igniteSupplier) {
- this(
- igniteSupplier,
- new TensorFlowServerManager(igniteSupplier),
- new TensorFlowClusterResolver(igniteSupplier)
- );
+ public TensorFlowClusterManager(Ignite ignite) {
+ assert ignite != null : "Ignite instance should not be null";
+
+ this.ignite = ignite;
+ this.srvProcMgr = new TensorFlowServerManager(ignite);
+ this.clusterRslvr = new TensorFlowClusterResolver(ignite, "TF", 10000, 1000);
+ this.cache = getOrCreateCache();
+ this.chiefRunners = new ConcurrentHashMap<>();
+ this.userScriptRunners = new ConcurrentHashMap<>();
}
/**
- * Constructs a new instance of TensorFlow cluster manager.
+ * Returns cluster by identifier.
*
- * @param igniteSupplier Ignite instance supplier.
- * @param srvProcMgr TensorFlow server manager.
- * @param clusterRslvr TensorFlow cluster resolver.
+ * @param clusterId Cluster identifier.
+ * @return TensorFlow cluster.
*/
- public <T extends Supplier<Ignite> & Serializable> TensorFlowClusterManager(T igniteSupplier,
- TensorFlowServerManager srvProcMgr, TensorFlowClusterResolver clusterRslvr) {
- assert igniteSupplier != null : "Ignite supplier should not be null";
- assert srvProcMgr != null : "TensorFlow server manager should not be null";
- assert clusterRslvr != null : "TensorFlow cluster resolver should not be null";
-
- this.igniteSupplier = igniteSupplier;
- this.srvProcMgr = srvProcMgr;
- this.clusterRslvr = clusterRslvr;
- }
-
- /** Initializes TensorFlow cluster manager and gets or creates correspondent caches. */
- public void init() {
- clusterRslvr.init();
-
- CacheConfiguration<String, TensorFlowCluster> cacheConfiguration = new CacheConfiguration<>();
- cacheConfiguration.setName(TF_CLUSTER_METADATA_CACHE_NAME);
- cacheConfiguration.setCacheMode(CacheMode.REPLICATED);
- cacheConfiguration.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
-
- Ignite ignite = igniteSupplier.get();
- cache = ignite.getOrCreateCache(cacheConfiguration);
+ public TensorFlowCluster getCluster(UUID clusterId) {
+ return cache.get(clusterId);
}
/**
- * Creates and starts a new TensorFlow cluster for the specified cache if it doesn't exist, otherwise returns
- * existing one.
+ * Creates and starts a new TensorFlow cluster for the specified cache.
*
- * @param upstreamCacheName Upstream cache name.
+ * @param clusterId Cluster identifier.
+ * @param jobArchive Job archive.
* @return TensorFlow cluster metadata.
*/
- public TensorFlowCluster getOrCreateCluster(String upstreamCacheName) {
- checkThatInitialized();
-
- Lock clusterMgrCacheLock = cache.lock(upstreamCacheName);
+ public TensorFlowCluster createCluster(UUID clusterId, TensorFlowJobArchive jobArchive,
+ Consumer<String> userScriptOut, Consumer<String> userScriptErr) {
+ Lock clusterMgrCacheLock = cache.lock(clusterId);
clusterMgrCacheLock.lock();
try {
- TensorFlowCluster cluster = cache.get(upstreamCacheName);
+ TensorFlowCluster cluster = cache.get(clusterId);
- if (cluster == null) {
- TensorFlowClusterSpec clusterSpec = clusterRslvr.resolveAndAcquirePorts(upstreamCacheName);
- cluster = startCluster(clusterSpec);
- cache.put(upstreamCacheName, cluster);
- }
+ if (cluster != null)
+ throw new IllegalStateException("Cluster is already created [clusterId=" + clusterId + "]");
+
+ TensorFlowClusterSpec clusterSpec = clusterRslvr.resolveAndAcquirePorts(jobArchive.getUpstreamCacheName());
+ cluster = startCluster(clusterId, clusterSpec, jobArchive, userScriptOut, userScriptErr);
+ cache.put(clusterId, cluster);
return cluster;
}
@@ -134,21 +127,21 @@ public class TensorFlowClusterManager implements Serializable {
/**
* Stops TensorFlow cluster.
*
- * @param upstreamCacheName Upstream cache name.
+ * @param clusterId TensorFlow cluster identifier.
*/
- public void stopClusterIfExists(String upstreamCacheName) {
- checkThatInitialized();
-
- Lock clusterMgrCacheLock = cache.lock(upstreamCacheName);
+ public void stopClusterIfExists(UUID clusterId) {
+ Lock clusterMgrCacheLock = cache.lock(clusterId);
clusterMgrCacheLock.lock();
try {
- TensorFlowCluster cluster = cache.get(upstreamCacheName);
+ TensorFlowCluster cluster = cache.get(clusterId);
if (cluster != null) {
+ stopChief(clusterId);
+ stopUserScript(clusterId);
srvProcMgr.stop(cluster.getProcesses(), true);
- clusterRslvr.freePorts(cluster.getSpec());
- cache.remove(upstreamCacheName);
+ clusterRslvr.releasePorts(cluster.getSpec());
+ cache.remove(clusterId);
}
}
finally {
@@ -159,44 +152,146 @@ public class TensorFlowClusterManager implements Serializable {
/** Destroys TensorFlow cluster manager and related caches. */
public void destroy() {
clusterRslvr.destroy();
-
- Ignite ignite = igniteSupplier.get();
ignite.destroyCache(TF_CLUSTER_METADATA_CACHE_NAME);
}
/**
* Starts TensorFlow cluster using the specified specification and returns metadata of the started cluster.
*
+ * @param clusterId Cluster identifier.
* @param spec TensorFlow cluster specification.
* @return TensorFlow cluster metadata.
*/
- private TensorFlowCluster startCluster(TensorFlowClusterSpec spec) {
- checkThatInitialized();
+ private TensorFlowCluster startCluster(UUID clusterId, TensorFlowClusterSpec spec, TensorFlowJobArchive jobArchive,
+ Consumer<String> userScriptOut, Consumer<String> userScriptErr) {
+ Map<String, List<TensorFlowServerAddressSpec>> jobs = spec.getJobs();
- List<TensorFlowServer> srvs = new ArrayList<>();
+ Map<UUID, List<UUID>> workerProcesses = startWorkers(spec, jobs.get(TensorFlowClusterResolver.WORKER_JOB_NAME));
- Map<String, List<TensorFlowServerAddressSpec>> jobs = spec.getJobs();
+ startChief(clusterId, spec);
+ startUserScript(clusterId, jobArchive, spec, userScriptOut, userScriptErr);
- for (String jobName : jobs.keySet()) {
- List<TensorFlowServerAddressSpec> tasks = jobs.get(jobName);
+ return new TensorFlowCluster(spec, workerProcesses);
+ }
+
+ /**
+ * Starts TensorFlow worker processes using the specified specification and returns identifiers of the started
+ * processes.
+ *
+ * @param spec TensorFlow cluster specification.
+ * @param tasks Worker tasks.
+ * @return Identifiers of the started processes.
+ */
+ private Map<UUID, List<UUID>> startWorkers(TensorFlowClusterSpec spec, List<TensorFlowServerAddressSpec> tasks) {
+ List<TensorFlowServer> srvs = new ArrayList<>();
+ if (tasks != null) {
for (int i = 0; i < tasks.size(); i++) {
- TensorFlowServer srvSpec = new TensorFlowServer(spec, jobName, i);
+ TensorFlowServer srvSpec = new TensorFlowServer(spec, TensorFlowClusterResolver.WORKER_JOB_NAME, i);
srvs.add(srvSpec);
}
}
- Map<UUID, List<UUID>> processes = srvProcMgr.start(srvs);
+ return srvProcMgr.start(srvs);
+ }
- return new TensorFlowCluster(spec, processes);
+ /**
+ * Starts chief process using the specified cluster specification.
+ *
+ * @param clusterId Cluster identifier.
+ * @param spec TensorFlow cluster specification.
+ */
+ private void startChief(UUID clusterId, TensorFlowClusterSpec spec) {
+ TensorFlowChiefRunner chiefRunner = new TensorFlowChiefRunner(
+ ignite,
+ Executors.newSingleThreadExecutor(
+ new CustomizableThreadFactory("tf-ch", true)
+ ),
+ spec,
+ System.out::println,
+ System.err::println
+ );
+
+ chiefRunner.start();
+
+ chiefRunners.put(clusterId, chiefRunner);
}
/**
- * Checks that the component has been initialized.
+ * Stops chief process.
+ *
+ * @param clusterId Cluster identifier.
*/
- private void checkThatInitialized() {
- if (cache == null)
- throw new IllegalStateException("TensorFlow Cluster Manager is not initialized");
+ private void stopChief(UUID clusterId) {
+ TensorFlowChiefRunner runner = chiefRunners.remove(clusterId);
+
+ if (runner != null)
+ runner.stop();
+ }
+
+ /**
+ * Starts user script processes using the specified job archive.
+ *
+ * @param clusterId Cluster identifier.
+ * @param jobArchive Job archive.
+ * @param clusterSpec Cluster specification.
+ */
+ private void startUserScript(UUID clusterId, TensorFlowJobArchive jobArchive, TensorFlowClusterSpec clusterSpec,
+ Consumer<String> out, Consumer<String> err) {
+ TensorFlowUserScriptRunner userScriptRunner = new TensorFlowUserScriptRunner(
+ ignite,
+ Executors.newSingleThreadExecutor(
+ new CustomizableThreadFactory("tf-us", true)
+ ),
+ jobArchive,
+ clusterSpec,
+ out,
+ err
+ );
+
+ userScriptRunner.start();
+
+ userScriptRunners.put(clusterId, userScriptRunner);
+ }
+
+ /**
+ * Stops user script process.
+ *
+ * @param clusterId Cluster identifier.
+ */
+ private void stopUserScript(UUID clusterId) {
+ TensorFlowUserScriptRunner runner = userScriptRunners.remove(clusterId);
+
+ if (runner != null)
+ runner.stop();
+ }
+
+ /**
+ * Checks if user script completed and returns result.
+ *
+ * @param clusterId Cluster identifier.
+ * @return {@code true} if user script completed, otherwise {@code false}.
+ */
+ public boolean isUserScriptCompleted(UUID clusterId) {
+ TensorFlowUserScriptRunner runner = userScriptRunners.get(clusterId);
+
+ return runner != null && runner.isCompleted();
+ }
+
+ /**
+ * Returns list of maintained TensorFlow clusters.
+ *
+ * @return List of maintained TensorFlow clusters.
+ */
+ public Map<UUID, TensorFlowCluster> getAllClusters() {
+ Map<UUID, TensorFlowCluster> res = new HashMap<>();
+
+ ScanQuery<UUID, TensorFlowCluster> qry = new ScanQuery<>();
+ QueryCursor<Cache.Entry<UUID, TensorFlowCluster>> cursor = cache.query(qry);
+ for (Cache.Entry<UUID, TensorFlowCluster> e : cursor)
+ res.put(e.getKey(), e.getValue());
+
+ return res;
}
/** */
@@ -204,8 +299,17 @@ public class TensorFlowClusterManager implements Serializable {
return srvProcMgr;
}
- /** */
- public IgniteCache<String, TensorFlowCluster> getCache() {
- return cache;
+ /**
+ * Returns existing cluster manager cache or creates a new one.
+ *
+ * @return Cluster manager cache.
+ */
+ private IgniteCache<UUID, TensorFlowCluster> getOrCreateCache() {
+ CacheConfiguration<UUID, TensorFlowCluster> cacheConfiguration = new CacheConfiguration<>();
+ cacheConfiguration.setName(TF_CLUSTER_METADATA_CACHE_NAME);
+ cacheConfiguration.setCacheMode(CacheMode.REPLICATED);
+ cacheConfiguration.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+ return ignite.getOrCreateCache(cacheConfiguration);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowJobArchive.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowJobArchive.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowJobArchive.java
new file mode 100644
index 0000000..953cf76
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/TensorFlowJobArchive.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.cluster;
+
+import java.io.Serializable;
+
+/**
+ * TensorFlow job archive that keeps archived working directory and command to be executed.
+ */
+public class TensorFlowJobArchive implements Serializable {
+ /** */
+ private static final long serialVersionUID = -5977231383594482459L;
+
+ /** Upstream cache name. */
+ private final String upstreamCacheName;
+
+ /** Archived working directory. */
+ private final byte[] data;
+
+ /** Command to be executed with arguments. */
+ private final String[] commands;
+
+ /**
+ * Constructs a new instance of TensorFlow job archive.
+ *
+ * @param upstreamCacheName Upstream cache name.
+ * @param data Archived working directory.
+ * @param commands Command to be executed with arguments.
+ */
+ public TensorFlowJobArchive(String upstreamCacheName, byte[] data, String[] commands) {
+ this.upstreamCacheName = upstreamCacheName;
+ this.data = data;
+ this.commands = commands;
+ }
+
+ /** */
+ public String getUpstreamCacheName() {
+ return upstreamCacheName;
+ }
+
+ /** */
+ public byte[] getData() {
+ return data;
+ }
+
+ /** */
+ public String[] getCommands() {
+ return commands;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowClusterSpec.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowClusterSpec.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowClusterSpec.java
index a053b8e..ea813ea 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowClusterSpec.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowClusterSpec.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import org.apache.ignite.Ignite;
/**
* TensorFlow cluster specification.
@@ -52,6 +53,44 @@ public class TensorFlowClusterSpec implements Serializable {
return this;
}
+ /**
+ * Formats cluster specification so that TensorFlow accepts it.
+ *
+ * @param ignite Ignite instance.
+ * @return Formatted cluster specification.
+ */
+ public String format(Ignite ignite) {
+ StringBuilder builder = new StringBuilder();
+
+ builder.append("{\n");
+
+ for (Map.Entry<String, List<TensorFlowServerAddressSpec>> entry : jobs.entrySet()) {
+ builder
+ .append("\t\"")
+ .append(entry.getKey())
+ .append("\" : [ ");
+
+ for (TensorFlowServerAddressSpec address : entry.getValue()) {
+ builder
+ .append("\n\t\t\"")
+ .append(address.format(ignite))
+ .append("\", ");
+ }
+
+ if (!entry.getValue().isEmpty())
+ builder.delete(builder.length() - 2, builder.length());
+
+ builder.append("\n\t],\n");
+ }
+
+ if (!jobs.isEmpty())
+ builder.delete(builder.length() - 2, builder.length() - 1);
+
+ builder.append('}');
+
+ return builder.toString();
+ }
+
/** */
public Map<String, List<TensorFlowServerAddressSpec>> getJobs() {
return jobs;
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowServerAddressSpec.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowServerAddressSpec.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowServerAddressSpec.java
index 196b166..727a030 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowServerAddressSpec.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/spec/TensorFlowServerAddressSpec.java
@@ -18,7 +18,9 @@
package org.apache.ignite.tensorflow.cluster.spec;
import java.io.Serializable;
+import java.util.Collection;
import java.util.UUID;
+import org.apache.ignite.Ignite;
/**
* TensorFlow server address specification.
@@ -47,6 +49,18 @@ public class TensorFlowServerAddressSpec implements Serializable {
this.port = port;
}
+ /**
+ * Formats Server Address specification so that TensorFlow accepts it.
+ *
+ * @param ignite Ignite instance.
+ * @return Formatted server address specification.
+ */
+ public String format(Ignite ignite) {
+ Collection<String> names = ignite.cluster().forNodeId(nodeId).hostNames();
+
+ return names.iterator().next() + ":" + port;
+ }
+
/** */
public UUID getNodeId() {
return nodeId;
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerManager.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerManager.java
index 192f619..ee87089 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerManager.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerManager.java
@@ -17,36 +17,32 @@
package org.apache.ignite.tensorflow.cluster.tfrunning;
-import java.io.Serializable;
-import java.util.function.Supplier;
-import org.apache.ignite.tensorflow.core.ProcessManager;
-import org.apache.ignite.tensorflow.core.ProcessManagerWrapper;
-import org.apache.ignite.tensorflow.core.pythonrunning.PythonProcess;
-import org.apache.ignite.tensorflow.core.pythonrunning.PythonProcessManager;
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowClusterSpec;
-import org.apache.ignite.tensorflow.cluster.spec.TensorFlowServerAddressSpec;
-import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
+import org.apache.ignite.tensorflow.cluster.spec.TensorFlowClusterSpec;
+import org.apache.ignite.tensorflow.cluster.spec.TensorFlowServerAddressSpec;
+import org.apache.ignite.tensorflow.core.ProcessManager;
+import org.apache.ignite.tensorflow.core.ProcessManagerWrapper;
+import org.apache.ignite.tensorflow.core.pythonrunning.PythonProcess;
+import org.apache.ignite.tensorflow.core.pythonrunning.PythonProcessManager;
/**
* TensorFlow server manager that allows to start, stop and make other actions with TensorFlow servers.
*/
public class TensorFlowServerManager extends ProcessManagerWrapper<PythonProcess, TensorFlowServer> {
- /** */
- private static final long serialVersionUID = 8355019934723445973L;
+ /** TensorFlow server script formatter. */
+ private static final TensorFlowServerScriptFormatter scriptFormatter = new TensorFlowServerScriptFormatter();
/**
* Constructs a new instance of TensorFlow server manager.
*
- * @param igniteSupplier Ignite instance supplier.
- * @param <T> Type of serializable supplier.
+ * @param ignite Ignite instance.
*/
- public <T extends Supplier<Ignite> & Serializable> TensorFlowServerManager(T igniteSupplier) {
- this(new PythonProcessManager(igniteSupplier));
+ public TensorFlowServerManager(Ignite ignite) {
+ this(new PythonProcessManager(ignite));
}
/**
@@ -61,7 +57,7 @@ public class TensorFlowServerManager extends ProcessManagerWrapper<PythonProcess
/** {@inheritDoc} */
@Override protected PythonProcess transformSpecification(TensorFlowServer spec) {
return new PythonProcess(
- formatPythonScript(spec),
+ scriptFormatter.format(spec, true, Ignition.ignite()),
getNode(spec)
);
}
@@ -80,88 +76,4 @@ public class TensorFlowServerManager extends ProcessManagerWrapper<PythonProcess
return addr.getNodeId();
}
-
- /**
- * Formats TensorFlow server specification so that it's available to be passed into а python script.
- *
- * @param spec TensorFlow server specification.
- * @return Formatted TensorFlow server specification.
- */
- private String formatPythonScript(TensorFlowServer spec) {
- StringBuilder builder = new StringBuilder();
-
- builder.append("import tensorflow as tf").append('\n');
- builder.append("cluster = tf.train.ClusterSpec(")
- .append(formatClusterSpec(spec.getClusterSpec()))
- .append(')')
- .append('\n');
- builder.append("server = tf.train.Server(cluster");
-
- if (spec.getJobName() != null)
- builder.append(", job_name=\"").append(spec.getJobName()).append('"');
-
- if (spec.getTaskIdx() != null)
- builder.append(", task_index=").append(spec.getTaskIdx());
-
- if (spec.getProto() != null)
- builder.append(", protocol=\"").append(spec.getProto()).append('"');
-
- builder.append(')').append('\n');
- builder.append("server.join()").append('\n');
-
- return builder.toString();
- }
-
- /**
- * Formats TensorFlow cluster specification so that it's available to be passed into а python script.
- *
- * @param spec TensorFlow cluster specification.
- * @return Formatted TensorFlow cluster specification.
- */
- public String formatClusterSpec(TensorFlowClusterSpec spec) {
- StringBuilder builder = new StringBuilder();
-
- builder.append("{\n");
-
- for (Map.Entry<String, List<TensorFlowServerAddressSpec>> entry : spec.getJobs().entrySet()) {
- builder
- .append("\t\"")
- .append(entry.getKey())
- .append("\" : [ ");
-
- for (TensorFlowServerAddressSpec address : entry.getValue()) {
- builder
- .append("\n\t\t\"")
- .append(formatAddressSpec(address))
- .append("\", ");
- }
-
- if (!entry.getValue().isEmpty())
- builder.delete(builder.length() - 2, builder.length());
-
- builder.append("\n\t],\n");
- }
-
- if (!spec.getJobs().isEmpty())
- builder.delete(builder.length() - 2, builder.length() - 1);
-
- builder.append('}');
-
- return builder.toString();
- }
-
- /**
- * Formats TensorFlow server address specification so that it's available to be passed into а python script.
- *
- * @param spec TensorFlow server address specification.
- * @return Formatted TensorFlow server address specification.
- */
- private String formatAddressSpec(TensorFlowServerAddressSpec spec) {
- UUID nodeId = spec.getNodeId();
-
- Ignite ignite = Ignition.localIgnite();
- Collection<String> names = ignite.cluster().forNodeId(nodeId).hostNames();
-
- return names.iterator().next() + ":" + spec.getPort();
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerScriptFormatter.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerScriptFormatter.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerScriptFormatter.java
new file mode 100644
index 0000000..0645964
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerScriptFormatter.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.cluster.tfrunning;
+
+import org.apache.ignite.Ignite;
+
+/**
+ * Utils class that helps to format Python script that starts TensorFlow server.
+ */
+public class TensorFlowServerScriptFormatter {
+ /**
+ * Formats TensorFlow server specification so that it's available to be passed into а python script.
+ *
+ * @param srv Server specification.
+ * @param join Joins server by default or not.
+ * @param ignite Ignite instance.
+ * @return Formatted TensorFlow server script.
+ */
+ public String format(TensorFlowServer srv, boolean join, Ignite ignite) {
+ StringBuilder builder = new StringBuilder();
+
+ builder.append("import tensorflow as tf").append('\n');
+ builder.append("from tensorflow.contrib.ignite import IgniteDataset").append("\n");
+ builder.append("cluster = tf.train.ClusterSpec(")
+ .append(srv.getClusterSpec().format(ignite))
+ .append(')')
+ .append('\n');
+
+ builder.append("server = tf.train.Server(cluster");
+
+ if (srv.getJobName() != null)
+ builder.append(", job_name=\"").append(srv.getJobName()).append('"');
+
+ if (srv.getTaskIdx() != null)
+ builder.append(", task_index=").append(srv.getTaskIdx());
+
+ if (srv.getProto() != null)
+ builder.append(", protocol=\"").append(srv.getProto()).append('"');
+
+ builder.append(')').append('\n');
+
+ if (join)
+ builder.append("server.join()").append('\n');
+
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/ClusterPortManager.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/ClusterPortManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/ClusterPortManager.java
index 78087ab..462752c 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/ClusterPortManager.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/ClusterPortManager.java
@@ -18,23 +18,33 @@
package org.apache.ignite.tensorflow.cluster.util;
import java.io.Serializable;
+import java.net.NetworkInterface;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
+import java.util.Enumeration;
+import java.util.List;
import java.util.UUID;
import java.util.concurrent.locks.Lock;
-import java.util.function.Supplier;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.cluster.ClusterGroupEmptyException;
import org.apache.ignite.configuration.CacheConfiguration;
/**
- * Cluster port manager that allows to reliably {@link #acquirePort(UUID)} and {@link #freePort(UUID, int)} on the
+ * Cluster port manager that allows to reliably {@link #acquirePort(UUID)} and {@link #releasePort(UUID, int)} on the
* cluster nodes.
*/
-public class ClusterPortManager implements Serializable {
- /** */
- private static final long serialVersionUID = -5116593574559007292L;
+public class ClusterPortManager {
+ /** Ignite instance. */
+ private final Ignite ignite;
+
+ /** Ignite logger. */
+ private final IgniteLogger log;
/** Port manager cache name. */
private final String portMgrCacheName;
@@ -45,11 +55,8 @@ public class ClusterPortManager implements Serializable {
/** Port range size. */
private final int cnt;
- /** Ignite instance supplier. */
- private final Supplier<Ignite> igniteSupplier;
-
/** Port manager cache */
- private transient IgniteCache<UUID, BitSet> cache;
+ private final IgniteCache<HostIdentifier, BitSet> cache;
/**
* Constructs a new instance of cluster port manager.
@@ -58,28 +65,19 @@ public class ClusterPortManager implements Serializable {
* @param from Port range from point.
* @param cnt Port range size.
*/
- public <T extends Supplier<Ignite> & Serializable> ClusterPortManager(String poolName, int from, int cnt,
- T igniteSupplier) {
+ public ClusterPortManager(Ignite ignite, String poolName, int from, int cnt) {
+ assert ignite != null : "Ignite instance should not be null";
assert poolName != null : "Pool name should not be null";
assert cnt >= 0 : "Count should not be negative";
assert from >= 0 && cnt + from <= 0xFFFF : "Port range should be between 0 and 65535";
- assert igniteSupplier != null : "Ignite supplier should not be null";
- this.portMgrCacheName = String.format("PORT_MANAGER_CACHE_%s", poolName);
+ this.ignite = ignite;
+ this.log = ignite.log().getLogger(ClusterPortManager.class);
+
+ this.portMgrCacheName = String.format("PORT_MANAGER_%s_CACHE", poolName);
this.from = from;
this.cnt = cnt;
- this.igniteSupplier = igniteSupplier;
- }
-
- /** Initializes port manager and creates or gets correspondent caches. */
- public void init() {
- CacheConfiguration<UUID, BitSet> cacheConfiguration = new CacheConfiguration<>();
- cacheConfiguration.setName(portMgrCacheName);
- cacheConfiguration.setCacheMode(CacheMode.REPLICATED);
- cacheConfiguration.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
-
- Ignite ignite = igniteSupplier.get();
- cache = ignite.getOrCreateCache(cacheConfiguration);
+ this.cache = getOrCreateCache();
}
/**
@@ -89,13 +87,16 @@ public class ClusterPortManager implements Serializable {
* @return Port to be acquired.
*/
public int acquirePort(UUID nodeId) {
- checkThatInitialized();
+ HostIdentifier hostId = getHostIdentifier(nodeId);
- Lock lock = cache.lock(nodeId);
+ if (hostId == null)
+ throw new IllegalStateException("Can't find node [nodeId=" + nodeId + "]");
+
+ Lock lock = cache.lock(hostId);
lock.lock();
try {
- BitSet ports = cache.get(nodeId);
+ BitSet ports = cache.get(hostId);
if (ports == null)
ports = new BitSet(cnt);
@@ -106,8 +107,9 @@ public class ClusterPortManager implements Serializable {
throw new IllegalStateException("No free ports in range [from=" + from + ", cnt=" + cnt + "]");
ports.set(free);
+ log.debug("Port acquired [nodeId=" + nodeId + ", port=" + (from + free) + "]");
- cache.put(nodeId, ports);
+ cache.put(hostId, ports);
return from + free;
}
@@ -117,27 +119,31 @@ public class ClusterPortManager implements Serializable {
}
/**
- * Frees acquired port on the specified node.
+ * Releases acquired port on the specified node.
*
* @param nodeId Node identifier.
* @param port Acquired port to be free.
*/
- public void freePort(UUID nodeId, int port) {
+ public void releasePort(UUID nodeId, int port) {
assert port - from >= 0 && port - from < cnt : "Port not in the range";
- checkThatInitialized();
+ HostIdentifier hostId = getHostIdentifier(nodeId);
+
+ if (hostId == null)
+ return;
- Lock lock = cache.lock(nodeId);
+ Lock lock = cache.lock(hostId);
lock.lock();
try {
- BitSet ports = cache.get(nodeId);
+ BitSet ports = cache.get(hostId);
if (ports != null) {
ports.clear(port - from);
+ log.debug("Port released [nodeId=" + nodeId + ", port=" + port + "]");
if (ports.isEmpty())
- cache.remove(nodeId);
+ cache.remove(hostId);
}
}
finally {
@@ -147,15 +153,97 @@ public class ClusterPortManager implements Serializable {
/** Destroys port manager and related caches. */
public void destroy() {
- Ignite ignite = igniteSupplier.get();
ignite.destroyCache(portMgrCacheName);
}
/**
- * Checks that the component has been initialized.
+ * Returns existed port pool cache or creates a new one.
+ *
+ * @return Port pool cache.
+ */
+ private IgniteCache<HostIdentifier, BitSet> getOrCreateCache() {
+ CacheConfiguration<HostIdentifier, BitSet> cacheConfiguration = new CacheConfiguration<>();
+ cacheConfiguration.setName(portMgrCacheName);
+ cacheConfiguration.setCacheMode(CacheMode.REPLICATED);
+ cacheConfiguration.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+ return ignite.getOrCreateCache(cacheConfiguration);
+ }
+
+ /**
+ * Returns host identifier by node identifier.
+ *
+ * @param nodeId Node identifier.
+ * @return Host identifier.
*/
- private void checkThatInitialized() {
- if (cache == null)
- throw new IllegalStateException("Cluster Port Manager is not initialized");
+ private HostIdentifier getHostIdentifier(UUID nodeId) {
+ try {
+ ClusterGroup grp = ignite.cluster().forNodeId(nodeId);
+
+ return ignite.compute(grp).call(() -> {
+ Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
+
+ List<byte[]> macAddrs = new ArrayList<>();
+
+ while (interfaces.hasMoreElements()) {
+ NetworkInterface netItf = interfaces.nextElement();
+ byte[] macAddr = netItf.getHardwareAddress();
+ macAddrs.add(macAddr);
+ }
+
+ return new HostIdentifier(macAddrs.toArray(new byte[macAddrs.size()][]));
+ });
+ }
+ catch (ClusterGroupEmptyException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Host identifier based on arrays of mac addresses of the host machine.
+ */
+ private static class HostIdentifier implements Serializable {
+ /** */
+ private static final long serialVersionUID = -7060231325908935162L;
+
+ /** Mac addresses. */
+ private final byte[][] macAddrs;
+
+ /**
+ * Constructs a new instance of host identifier.
+ *
+ * @param macAddrs Mac addresses.
+ */
+ public HostIdentifier(byte[][] macAddrs) {
+ this.macAddrs = macAddrs;
+ }
+
+ /** */
+ public byte[][] getMacAddrs() {
+ return macAddrs;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ HostIdentifier that = (HostIdentifier)o;
+ if (macAddrs.length != that.macAddrs.length)
+ return false;
+
+ for (int i = 0; i < macAddrs.length; i++)
+ if (!Arrays.equals(macAddrs[i], that.macAddrs[i]))
+ return false;
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Arrays.hashCode(macAddrs);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowChiefRunner.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowChiefRunner.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowChiefRunner.java
new file mode 100644
index 0000000..6998681
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowChiefRunner.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.cluster.util;
+
+import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.tensorflow.cluster.spec.TensorFlowClusterSpec;
+import org.apache.ignite.tensorflow.cluster.tfrunning.TensorFlowServer;
+import org.apache.ignite.tensorflow.cluster.tfrunning.TensorFlowServerScriptFormatter;
+import org.apache.ignite.tensorflow.core.pythonrunning.PythonProcessBuilderSupplier;
+import org.apache.ignite.tensorflow.core.util.AsyncNativeProcessRunner;
+import org.apache.ignite.tensorflow.core.util.NativeProcessRunner;
+
+/**
+ * Utils class that helps to start and stop chief process.
+ */
+public class TensorFlowChiefRunner extends AsyncNativeProcessRunner {
+ /** Ignite instance. */
+ private final Ignite ignite;
+
+ /** TensorFlow cluster specification. */
+ private final TensorFlowClusterSpec spec;
+
+ /** Output stream data consumer. */
+ private final Consumer<String> out;
+
+ /** Error stream data consumer. */
+ private final Consumer<String> err;
+
+ /**
+ * Constructs a new instance of TensorFlow chief runner.
+ *
+ * @param ignite Ignite instance.
+ * @param executor Executor to be used in {@link AsyncNativeProcessRunner}.
+ * @param spec TensorFlow cluster specification.
+ * @param out Output stream data consumer.
+ * @param err Error stream data consumer.
+ */
+ public TensorFlowChiefRunner(Ignite ignite, ExecutorService executor, TensorFlowClusterSpec spec,
+ Consumer<String> out, Consumer<String> err) {
+ super(ignite, executor);
+ this.ignite = ignite;
+ this.spec = spec;
+ this.out = out;
+ this.err = err;
+ }
+
+ /** {@inheritDoc} */
+ @Override public NativeProcessRunner doBefore() {
+ TensorFlowServer srv = new TensorFlowServer(spec, TensorFlowClusterResolver.CHIEF_JOB_NAME, 0);
+
+ return new NativeProcessRunner(
+ new PythonProcessBuilderSupplier(true).get(),
+ new TensorFlowServerScriptFormatter().format(srv, true, ignite),
+ out,
+ err
+ );
+ }
+
+ /** {@inheritDoc} */
+ @Override public void doAfter() {
+ // Do nothing.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowClusterResolver.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowClusterResolver.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowClusterResolver.java
index e1d7d57..846af71 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowClusterResolver.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowClusterResolver.java
@@ -17,9 +17,7 @@
package org.apache.ignite.tensorflow.cluster.util;
-import java.io.Serializable;
import java.util.UUID;
-import java.util.function.Supplier;
import org.apache.ignite.Ignite;
import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cluster.ClusterNode;
@@ -29,32 +27,32 @@ import org.apache.ignite.tensorflow.cluster.spec.TensorFlowServerAddressSpec;
/**
* TensorFlow cluster resolver based on Ignite Cache affinity.
*/
-public class TensorFlowClusterResolver implements Serializable {
- /** */
- private static final long serialVersionUID = 631456775167710173L;
+public class TensorFlowClusterResolver {
+ /** TensorFlow worker job name. */
+ public static final String WORKER_JOB_NAME = "worker";
+
+ /** TensorFlow chief job name. */
+ public static final String CHIEF_JOB_NAME = "chief";
+
+ /** Ignite instance. */
+ private final Ignite ignite;
/** Cluster port manager. */
private final ClusterPortManager portMgr;
- /** Ignite instance supplier. */
- private final Supplier<Ignite> igniteSupplier;
-
/**
* Constructs a new instance of TensorFlow cluster resolver.
*
- * @param igniteSupplier Ignite instance supplier.
- * @param <T> Type of serializable supplier.
+ * @param ignite Ignite instance.
*/
- public <T extends Supplier<Ignite> & Serializable> TensorFlowClusterResolver(T igniteSupplier) {
- assert igniteSupplier != null : "Ignite supplier should not be null";
-
- this.igniteSupplier = igniteSupplier;
- this.portMgr = new ClusterPortManager("TF_POOL", 10000, 100, igniteSupplier);
- }
-
- /** Initializes TensorFlow cluster resolver. */
- public void init() {
- portMgr.init();
+ public TensorFlowClusterResolver(Ignite ignite, String portPoolName, int portFrom, int portCnt) {
+ assert ignite != null : "Ignite instance should not be null";
+ assert portPoolName != null : "Port pool name should not be null";
+ assert portFrom >= 0 : "Port count should not be negative";
+ assert portCnt >= 0 && portCnt + portFrom <= 0xFFFF : "Port range should be between 0 and 65535";
+
+ this.ignite = ignite;
+ this.portMgr = new ClusterPortManager(ignite, portPoolName, portFrom, portCnt);
}
/**
@@ -64,38 +62,60 @@ public class TensorFlowClusterResolver implements Serializable {
* @return TensorFlow cluster specification.
*/
public TensorFlowClusterSpec resolveAndAcquirePorts(String upstreamCacheName) {
- Ignite ignite = igniteSupplier.get();
- Affinity<?> affinity = ignite.affinity(upstreamCacheName);
-
- int parts = affinity.partitions();
-
TensorFlowClusterSpec spec = new TensorFlowClusterSpec();
- for (int part = 0; part < parts; part++) {
- ClusterNode node = affinity.mapPartitionToNode(part);
- UUID nodeId = node.id();
-
- int port = portMgr.acquirePort(nodeId);
-
- spec.addTask("WORKER", nodeId, port);
- }
+ resolveAndAcquirePortsForWorkers(spec, upstreamCacheName);
+ resolveAndAcquirePortsForChief(spec);
return spec;
}
/**
- * Frees ports acquired for the given cluster specification.
+ * Releases ports acquired for the given cluster specification.
*
* @param spec TensorFlow cluster specification.
*/
- public void freePorts(TensorFlowClusterSpec spec) {
+ public void releasePorts(TensorFlowClusterSpec spec) {
for (String jobName : spec.getJobs().keySet())
for (TensorFlowServerAddressSpec address : spec.getJobs().get(jobName))
- portMgr.freePort(address.getNodeId(), address.getPort());
+ portMgr.releasePort(address.getNodeId(), address.getPort());
}
/** Destroys TensorFlow cluster resolver. */
public void destroy() {
portMgr.destroy();
}
+
+ /**
+ * Resolves TensorFlow cluster worker jobs and acquires ports.
+ *
+ * @param spec TensorFlow cluster specification.
+ * @param upstreamCacheName Upstream cache name.
+ */
+ private void resolveAndAcquirePortsForWorkers(TensorFlowClusterSpec spec, String upstreamCacheName) {
+ Affinity<?> affinity = ignite.affinity(upstreamCacheName);
+ int parts = affinity.partitions();
+
+ for (int part = 0; part < parts; part++) {
+ ClusterNode node = affinity.mapPartitionToNode(part);
+ UUID nodeId = node.id();
+
+ int port = portMgr.acquirePort(nodeId);
+
+ spec.addTask(WORKER_JOB_NAME, nodeId, port);
+ }
+ }
+
+ /**
+ * Resolves TensorFlow cluster chief job and acquires ports.
+ *
+ * @param spec TensorFlow cluster specification.
+ */
+ private void resolveAndAcquirePortsForChief(TensorFlowClusterSpec spec) {
+ ClusterNode chiefNode = ignite.cluster().localNode();
+ UUID chiefNodeId = chiefNode.id();
+ int chiefPort = portMgr.acquirePort(chiefNodeId);
+
+ spec.addTask(CHIEF_JOB_NAME, chiefNodeId, chiefPort);
+ }
}
[12/50] [abbrv] ignite git commit: IGNITE-8999 Web Console: Fixed
issue with missing errorParser on paragraph creation.
Posted by av...@apache.org.
IGNITE-8999 Web Console: Fixed issue with missing errorParser on paragraph creation.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bea6fe31
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bea6fe31
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bea6fe31
Branch: refs/heads/ignite-8446
Commit: bea6fe31125f907c57446582292b878ef7d6d994
Parents: f6c67e8
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Tue Jul 31 18:35:20 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Jul 31 18:35:20 2018 +0700
----------------------------------------------------------------------
.../components/queries-notebook/controller.js | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bea6fe31/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
index 132f7f6..c887f4d 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
@@ -66,7 +66,7 @@ const _fullColName = (col) => {
let paragraphId = 0;
class Paragraph {
- constructor($animate, $timeout, JavaTypes, paragraph, errorParser) {
+ constructor($animate, $timeout, JavaTypes, errorParser, paragraph) {
const self = this;
self.id = 'paragraph-' + paragraphId++;
@@ -957,6 +957,10 @@ export class NotebookCtrl {
.subscribe();
};
+ const _newParagraph = (paragraph) => {
+ return new Paragraph($animate, $timeout, JavaTypes, errorParser, paragraph);
+ };
+
Notebook.find($state.params.noteId)
.then((notebook) => {
$scope.notebook = _.cloneDeep(notebook);
@@ -969,8 +973,7 @@ export class NotebookCtrl {
if (!$scope.notebook.paragraphs)
$scope.notebook.paragraphs = [];
- $scope.notebook.paragraphs = _.map($scope.notebook.paragraphs,
- (paragraph) => new Paragraph($animate, $timeout, JavaTypes, paragraph, this.errorParser));
+ $scope.notebook.paragraphs = _.map($scope.notebook.paragraphs, (p) => _newParagraph(p));
if (_.isEmpty($scope.notebook.paragraphs))
$scope.addQuery();
@@ -1042,7 +1045,7 @@ export class NotebookCtrl {
ActivitiesData.post({ action: '/queries/add/query' });
- const paragraph = new Paragraph($animate, $timeout, JavaTypes, {
+ const paragraph = _newParagraph({
name: 'Query' + (sz === 0 ? '' : sz),
query: '',
pageSize: $scope.pageSizes[1],
@@ -1071,7 +1074,7 @@ export class NotebookCtrl {
ActivitiesData.post({ action: '/queries/add/scan' });
- const paragraph = new Paragraph($animate, $timeout, JavaTypes, {
+ const paragraph = _newParagraph({
name: 'Scan' + (sz === 0 ? '' : sz),
query: '',
pageSize: $scope.pageSizes[1],
[28/50] [abbrv] ignite git commit: IGNITE-7752: Update Ignite
KafkaStreamer to use new consumer. - Fixes #4363.
Posted by av...@apache.org.
IGNITE-7752: Update Ignite KafkaStreamer to use new consumer. - Fixes #4363.
Signed-off-by: shroman <rs...@yahoo.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/44391198
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/44391198
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/44391198
Branch: refs/heads/ignite-8446
Commit: 443911988d246a2cc62abc450758a3702552046a
Parents: ff89193
Author: shroman <rs...@yahoo.com>
Authored: Wed Aug 1 16:53:28 2018 +0900
Committer: shroman <rs...@yahoo.com>
Committed: Wed Aug 1 16:53:28 2018 +0900
----------------------------------------------------------------------
modules/kafka/pom.xml | 12 +-
.../ignite/stream/kafka/KafkaStreamer.java | 173 ++++++++++---------
.../kafka/KafkaIgniteStreamerSelfTest.java | 39 ++---
.../ignite/stream/kafka/TestKafkaBroker.java | 18 +-
4 files changed, 121 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/44391198/modules/kafka/pom.xml
----------------------------------------------------------------------
diff --git a/modules/kafka/pom.xml b/modules/kafka/pom.xml
index f015d21..18ffcaa 100644
--- a/modules/kafka/pom.xml
+++ b/modules/kafka/pom.xml
@@ -20,7 +20,8 @@
<!--
POM file.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -43,7 +44,7 @@
<dependency>
<groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.11</artifactId>
+ <artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
</dependency>
@@ -55,6 +56,13 @@
<dependency>
<groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.11</artifactId>
+ <version>${kafka.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
<artifactId>connect-runtime</artifactId>
<version>${kafka.version}</version>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/ignite/blob/44391198/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java
----------------------------------------------------------------------
diff --git a/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java b/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java
index ed3530b..bdbd916 100644
--- a/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java
+++ b/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java
@@ -17,68 +17,63 @@
package org.apache.ignite.stream.kafka;
-import java.util.HashMap;
+import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-import kafka.consumer.ConsumerConfig;
-import kafka.consumer.ConsumerIterator;
-import kafka.consumer.KafkaStream;
-import kafka.javaapi.consumer.ConsumerConnector;
-import kafka.message.MessageAndMetadata;
+import java.util.stream.IntStream;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.stream.StreamAdapter;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.errors.WakeupException;
/**
* Server that subscribes to topic messages from Kafka broker and streams its to key-value pairs into
* {@link IgniteDataStreamer} instance.
* <p>
* Uses Kafka's High Level Consumer API to read messages from Kafka.
- *
- * @see <a href="https://cwiki.apache.org/confluence/display/KAFKA/Consumer+Group+Example">Consumer Consumer Group
- * Example</a>
*/
-public class KafkaStreamer<K, V> extends StreamAdapter<MessageAndMetadata<byte[], byte[]>, K, V> {
- /** Retry timeout. */
- private static final long DFLT_RETRY_TIMEOUT = 10000;
+public class KafkaStreamer<K, V> extends StreamAdapter<ConsumerRecord, K, V> {
+ /** Default polling timeout. */
+ private final static long DFLT_TIMEOUT = 100;
/** Logger. */
private IgniteLogger log;
- /** Executor used to submit kafka streams. */
+ /** Polling tasks executor. */
private ExecutorService executor;
- /** Topic. */
- private String topic;
+ /** Topics. */
+ private List<String> topics;
- /** Number of threads to process kafka streams. */
+ /** Number of threads. */
private int threads;
/** Kafka consumer config. */
- private ConsumerConfig consumerCfg;
-
- /** Kafka consumer connector. */
- private ConsumerConnector consumer;
+ private Properties consumerCfg;
- /** Retry timeout. */
- private long retryTimeout = DFLT_RETRY_TIMEOUT;
+ /** Polling timeout. */
+ private long timeout = DFLT_TIMEOUT;
- /** Stopped. */
- private volatile boolean stopped;
+ /** Kafka consumer tasks. */
+ private final List<ConsumerTask> consumerTasks = new ArrayList<>();
/**
- * Sets the topic name.
+ * Sets the topic names.
*
- * @param topic Topic name.
+ * @param topics Topic names.
*/
- public void setTopic(String topic) {
- this.topic = topic;
+ public void setTopic(List<String> topics) {
+ this.topics = topics;
}
/**
@@ -95,19 +90,19 @@ public class KafkaStreamer<K, V> extends StreamAdapter<MessageAndMetadata<byte[]
*
* @param consumerCfg Consumer configuration.
*/
- public void setConsumerConfig(ConsumerConfig consumerCfg) {
+ public void setConsumerConfig(Properties consumerCfg) {
this.consumerCfg = consumerCfg;
}
/**
- * Sets the retry timeout.
+ * Sets the polling timeout for Kafka tasks.
*
- * @param retryTimeout Retry timeout.
+ * @param timeout Timeout.
*/
- public void setRetryTimeout(long retryTimeout) {
- A.ensure(retryTimeout > 0, "retryTimeout > 0");
+ public void setTimeout(long timeout) {
+ A.ensure(timeout > 0, "timeout > 0");
- this.retryTimeout = retryTimeout;
+ this.timeout = timeout;
}
/**
@@ -118,7 +113,7 @@ public class KafkaStreamer<K, V> extends StreamAdapter<MessageAndMetadata<byte[]
public void start() {
A.notNull(getStreamer(), "streamer");
A.notNull(getIgnite(), "ignite");
- A.notNull(topic, "topic");
+ A.notNull(topics, "topics");
A.notNull(consumerCfg, "kafka consumer config");
A.ensure(threads > 0, "threads > 0");
A.ensure(null != getSingleTupleExtractor() || null != getMultipleTupleExtractor(),
@@ -126,65 +121,20 @@ public class KafkaStreamer<K, V> extends StreamAdapter<MessageAndMetadata<byte[]
log = getIgnite().log();
- consumer = kafka.consumer.Consumer.createJavaConsumerConnector(consumerCfg);
-
- Map<String, Integer> topicCntMap = new HashMap<>();
-
- topicCntMap.put(topic, threads);
-
- Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCntMap);
-
- List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(topic);
-
- // Now launch all the consumer threads.
executor = Executors.newFixedThreadPool(threads);
- stopped = false;
+ IntStream.range(0, threads).forEach(i -> consumerTasks.add(new ConsumerTask(consumerCfg)));
- // Now create an object to consume the messages.
- for (final KafkaStream<byte[], byte[]> stream : streams) {
- executor.execute(new Runnable() {
- @Override public void run() {
- while (!stopped) {
- try {
- MessageAndMetadata<byte[], byte[]> msg;
-
- for (ConsumerIterator<byte[], byte[]> it = stream.iterator(); it.hasNext() && !stopped; ) {
- msg = it.next();
-
- try {
- addMessage(msg);
- }
- catch (Exception e) {
- U.error(log, "Message is ignored due to an error [msg=" + msg + ']', e);
- }
- }
- }
- catch (Exception e) {
- U.error(log, "Message can't be consumed from stream. Retry after " +
- retryTimeout + " ms.", e);
-
- try {
- Thread.sleep(retryTimeout);
- }
- catch (InterruptedException ignored) {
- // No-op.
- }
- }
- }
- }
- });
- }
+ for (ConsumerTask task : consumerTasks)
+ executor.submit(task);
}
/**
* Stops streamer.
*/
public void stop() {
- stopped = true;
-
- if (consumer != null)
- consumer.shutdown();
+ for (ConsumerTask task : consumerTasks)
+ task.stop();
if (executor != null) {
executor.shutdown();
@@ -200,4 +150,55 @@ public class KafkaStreamer<K, V> extends StreamAdapter<MessageAndMetadata<byte[]
}
}
}
+
+ /** Polling task. */
+ class ConsumerTask implements Callable<Void> {
+ /** Kafka consumer. */
+ private final KafkaConsumer<?, ?> consumer;
+
+ /** Stopped. */
+ private volatile boolean stopped;
+
+ /** Constructor. */
+ public ConsumerTask(Properties consumerCfg) {
+ this.consumer = new KafkaConsumer<>(consumerCfg);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Void call() {
+ consumer.subscribe(topics);
+
+ try {
+ while (!stopped) {
+ for (ConsumerRecord record : consumer.poll(timeout)) {
+ try {
+ addMessage(record);
+ }
+ catch (Exception e) {
+ U.error(log, "Record is ignored due to an error [record = " + record + ']', e);
+ }
+ }
+ }
+ }
+ catch (WakeupException we) {
+ log.info("Consumer is being stopped.");
+ }
+ catch (KafkaException ke) {
+ log.error("Kafka error", ke);
+ }
+ finally {
+ consumer.close();
+ }
+
+ return null;
+ }
+
+ /** Stops the polling task. */
+ public void stop() {
+ stopped = true;
+
+ if (consumer != null)
+ consumer.wakeup();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/44391198/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/KafkaIgniteStreamerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/KafkaIgniteStreamerSelfTest.java b/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/KafkaIgniteStreamerSelfTest.java
index 00cb4fc..48d4a8d 100644
--- a/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/KafkaIgniteStreamerSelfTest.java
+++ b/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/KafkaIgniteStreamerSelfTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.stream.kafka;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -27,17 +28,16 @@ import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import kafka.consumer.ConsumerConfig;
-import kafka.message.MessageAndMetadata;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.events.CacheEvent;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.lang.IgniteBiPredicate;
-import org.apache.ignite.stream.StreamMultipleTupleExtractor;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.serialization.StringDeserializer;
import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_PUT;
@@ -167,23 +167,22 @@ public class KafkaIgniteStreamerSelfTest extends GridCommonAbstractTest {
kafkaStmr.setStreamer(stmr);
// Set the topic.
- kafkaStmr.setTopic(topic);
+ kafkaStmr.setTopic(Arrays.asList(topic));
// Set the number of threads.
kafkaStmr.setThreads(4);
// Set the consumer configuration.
kafkaStmr.setConsumerConfig(
- createDefaultConsumerConfig(embeddedBroker.getZookeeperAddress(), "groupX"));
+ createDefaultConsumerConfig(embeddedBroker.getBrokerAddress(), "groupX"));
kafkaStmr.setMultipleTupleExtractor(
- new StreamMultipleTupleExtractor<MessageAndMetadata<byte[], byte[]>, String, String>() {
- @Override public Map<String, String> extract(MessageAndMetadata<byte[], byte[]> msg) {
+ record -> {
Map<String, String> entries = new HashMap<>();
try {
- String key = new String(msg.key());
- String val = new String(msg.message());
+ String key = (String)record.key();
+ String val = (String)record.value();
// Convert the message into number of cache entries with same key or dynamic key from actual message.
// For now using key as cache entry key and value as cache entry value - for test purpose.
@@ -194,8 +193,7 @@ public class KafkaIgniteStreamerSelfTest extends GridCommonAbstractTest {
}
return entries;
- }
- });
+ });
// Start kafka streamer.
kafkaStmr.start();
@@ -227,23 +225,22 @@ public class KafkaIgniteStreamerSelfTest extends GridCommonAbstractTest {
/**
* Creates default consumer config.
*
- * @param zooKeeper ZooKeeper address <server:port>.
+ * @param servers Bootstrap servers' address in the form of <server:port;server:port>.
* @param grpId Group Id for kafka subscriber.
* @return Kafka consumer configuration.
*/
- private ConsumerConfig createDefaultConsumerConfig(String zooKeeper, String grpId) {
- A.notNull(zooKeeper, "zookeeper");
+ private Properties createDefaultConsumerConfig(String servers, String grpId) {
+ A.notNull(servers, "bootstrap servers");
A.notNull(grpId, "groupId");
Properties props = new Properties();
- props.put("zookeeper.connect", zooKeeper);
- props.put("group.id", grpId);
- props.put("zookeeper.session.timeout.ms", "400");
- props.put("zookeeper.sync.time.ms", "200");
- props.put("auto.commit.interval.ms", "1000");
- props.put("auto.offset.reset", "smallest");
+ props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
+ props.put(ConsumerConfig.GROUP_ID_CONFIG, grpId);
+ props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
+ props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
- return new ConsumerConfig(props);
+ return props;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/44391198/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/TestKafkaBroker.java
----------------------------------------------------------------------
diff --git a/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/TestKafkaBroker.java b/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/TestKafkaBroker.java
index 803d55e..4f0d1d3 100644
--- a/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/TestKafkaBroker.java
+++ b/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/TestKafkaBroker.java
@@ -36,7 +36,9 @@ import org.apache.curator.test.TestingServer;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.serialization.StringSerializer;
import scala.Tuple2;
/**
@@ -202,15 +204,6 @@ public class TestKafkaBroker {
}
/**
- * Obtains Zookeeper address.
- *
- * @return Zookeeper address.
- */
- public String getZookeeperAddress() {
- return BROKER_HOST + ":" + ZK_PORT;
- }
-
- /**
* Obtains producer config.
*
* @return Kafka Producer config.
@@ -218,9 +211,10 @@ public class TestKafkaBroker {
private Properties getProducerConfig() {
Properties props = new Properties();
- props.put("bootstrap.servers", getBrokerAddress());
- props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
- props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, getBrokerAddress());
+ props.put(ProducerConfig.CLIENT_ID_CONFIG, "KafkaTestProducer");
+ props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
return props;
}
[25/50] [abbrv] ignite git commit: IGNITE-7979 Web Console: Removed
delete button from "Notebook" screen,
as of this function moved to "Notebooks" screen.
Posted by av...@apache.org.
IGNITE-7979 Web Console: Removed delete button from "Notebook" screen, as of this function moved to "Notebooks" screen.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/19f9196c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/19f9196c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/19f9196c
Branch: refs/heads/ignite-8446
Commit: 19f9196c9b1727b3317052309c53fcbb43206996
Parents: 8ea35e7
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Wed Aug 1 12:29:53 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Aug 1 12:29:53 2018 +0700
----------------------------------------------------------------------
.../page-queries/components/queries-notebook/template.tpl.pug | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/19f9196c/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
index ab188fa..713f83e 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
@@ -124,7 +124,6 @@ mixin query-actions
i.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.executionInProgress(true)')
span.tipLabelExecute Execute on selected node
-
a.btn.btn-default(ng-disabled='!queryAvailable(paragraph)' ng-click='explain(paragraph)' data-placement='bottom' bs-tooltip='' data-title='{{queryTooltip(paragraph, "explain query")}}') Explain
mixin table-result-heading-query
@@ -347,7 +346,6 @@ div(ng-if='notebook')
label {{notebook.name}}
.btn-group(ng-if='!demo')
+btn-toolbar('fa-pencil', 'notebook.edit = true;notebook.editName = notebook.name', 'Rename notebook')
- +btn-toolbar('fa-trash', 'removeNotebook(notebook)', '{{$ctrl.isDemo ? "Demo notebook cannot be deleted" : "Remove notebook"}}')
h1(ng-show='notebook.edit')
input.form-control(ng-model='notebook.editName' required ignite-on-enter='renameNotebook(notebook.editName)' ignite-on-escape='notebook.edit = false;')
[49/50] [abbrv] ignite git commit: IGNITE-9179 Baseline: Added
separate task for collecting info about baseline.
Posted by av...@apache.org.
IGNITE-9179 Baseline: Added separate task for collecting info about baseline.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ff7372b7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ff7372b7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ff7372b7
Branch: refs/heads/ignite-8446
Commit: ff7372b7128df3b57ee02b735cc3010645697b7a
Parents: f825dfc
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Fri Aug 3 16:31:41 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Aug 3 16:31:41 2018 +0700
----------------------------------------------------------------------
.../visor/baseline/VisorBaselineTask.java | 2 +-
.../visor/baseline/VisorBaselineViewTask.java | 73 ++++++++++++++++++++
.../internal/visor/tx/VisorTxTaskArg.java | 1 -
.../internal/visor/tx/VisorTxTaskResult.java | 7 --
4 files changed, 74 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ff7372b7/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
index 721b4b3..3c00452 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
@@ -34,7 +34,7 @@ import org.apache.ignite.internal.visor.VisorOneNodeTask;
import org.jetbrains.annotations.Nullable;
/**
- * Task that will collect baseline topology information.
+ * Task that will collect information about baseline topology and can change its state.
*/
@GridInternal
public class VisorBaselineTask extends VisorOneNodeTask<VisorBaselineTaskArg, VisorBaselineTaskResult> {
http://git-wip-us.apache.org/repos/asf/ignite/blob/ff7372b7/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineViewTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineViewTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineViewTask.java
new file mode 100644
index 0000000..472b907
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineViewTask.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.baseline;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task that will collect information about baseline topology.
+ */
+@GridInternal
+public class VisorBaselineViewTask extends VisorOneNodeTask<Void, VisorBaselineTaskResult> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override protected VisorBaselineViewJob job(Void arg) {
+ return new VisorBaselineViewJob(arg, debug);
+ }
+
+ /**
+ * Job that will collect baseline topology information.
+ */
+ private static class VisorBaselineViewJob extends VisorJob<Void, VisorBaselineTaskResult> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * @param arg Formal job argument.
+ * @param debug Debug flag.
+ */
+ private VisorBaselineViewJob(Void arg, boolean debug) {
+ super(arg, debug);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected VisorBaselineTaskResult run(@Nullable Void arg) throws IgniteException {
+ IgniteClusterEx cluster = ignite.cluster();
+
+ return new VisorBaselineTaskResult(
+ ignite.cluster().active(),
+ cluster.topologyVersion(),
+ cluster.currentBaselineTopology(),
+ cluster.forServers().nodes()
+ );
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(VisorBaselineViewJob.class, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ff7372b7/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskArg.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskArg.java
index 80bf8b5..6f157e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskArg.java
@@ -24,7 +24,6 @@ import java.util.List;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.visor.VisorDataTransferObject;
-import org.apache.ignite.internal.visor.baseline.VisorBaselineTask;
import org.apache.ignite.transactions.TransactionState;
import org.jetbrains.annotations.Nullable;
http://git-wip-us.apache.org/repos/asf/ignite/blob/ff7372b7/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskResult.java
index 4864a77..1d49fd9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskResult.java
@@ -20,17 +20,10 @@ package org.apache.ignite.internal.visor.tx;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
-import java.util.Collection;
import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import org.apache.ignite.cluster.BaselineNode;
-import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.visor.VisorDataTransferObject;
-import org.apache.ignite.internal.visor.baseline.VisorBaselineNode;
-import org.apache.ignite.internal.visor.baseline.VisorBaselineTask;
/**
* Result for {@link VisorTxTask}.
[31/50] [abbrv] ignite git commit: IGNITE-8900 SqlFieldsQuery
provides incorrect result when item size exceeds page size.
Posted by av...@apache.org.
IGNITE-8900 SqlFieldsQuery provides incorrect result when item size exceeds page size.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f1ecbbc8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f1ecbbc8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f1ecbbc8
Branch: refs/heads/ignite-8446
Commit: f1ecbbc87d6a42dad960cda09cbc43992ff286e1
Parents: 458480c
Author: ilantukh <il...@gridgain.com>
Authored: Wed Aug 1 16:31:15 2018 +0300
Committer: Dmitriy Pavlov <dp...@apache.org>
Committed: Wed Aug 1 16:31:15 2018 +0300
----------------------------------------------------------------------
.../delta/DataPageInsertFragmentRecord.java | 2 +-
.../cache/IgniteCacheOffheapManagerImpl.java | 3 +
.../persistence/freelist/AbstractFreeList.java | 2 +-
.../persistence/freelist/CacheFreeListImpl.java | 11 ++
.../persistence/tree/io/AbstractDataPageIO.java | 23 ++-
.../processors/cache/BigEntryQueryTest.java | 150 +++++++++++++++++++
.../IgniteBinaryCacheQueryTestSuite.java | 2 +
7 files changed, 177 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1ecbbc8/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
index 5324d56..2b02bb57 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
@@ -57,7 +57,7 @@ public class DataPageInsertFragmentRecord extends PageDeltaRecord {
@Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
AbstractDataPageIO io = PageIO.getPageIO(pageAddr);
- io.addRowFragment(pageAddr, payload, lastLink, pageMem.pageSize());
+ io.addRowFragment(PageIO.getPageId(pageAddr), pageAddr, payload, lastLink, pageMem.pageSize());
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1ecbbc8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 10281d6..5482b3a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -1386,6 +1386,9 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
assert oldRow == null || oldRow.cacheId() == cacheId : oldRow;
+ if (key.partition() == -1)
+ key.partition(partId);
+
DataRow dataRow = new DataRow(key, val, ver, partId, expireTime, cacheId);
CacheObjectContext coCtx = cctx.cacheObjectContext();
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1ecbbc8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
index c1a48bb..bcedd8c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
@@ -233,7 +233,7 @@ public abstract class AbstractFreeList<T extends Storable> extends PagesList imp
// Read last link before the fragment write, because it will be updated there.
long lastLink = row.link();
- int payloadSize = io.addRowFragment(pageMem, pageAddr, row, written, rowSize, pageSize());
+ int payloadSize = io.addRowFragment(pageMem, pageId, pageAddr, row, written, rowSize, pageSize());
assert payloadSize > 0 : payloadSize;
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1ecbbc8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
index dc0c92e..4aa9d88 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.cache.persistence.freelist;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
@@ -26,6 +27,7 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractD
import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.util.typedef.internal.U;
/**
* FreeList implementation for cache.
@@ -53,6 +55,15 @@ public class CacheFreeListImpl extends AbstractFreeList<CacheDataRow> {
}
/** {@inheritDoc} */
+ @Override public void insertDataRow(CacheDataRow row) throws IgniteCheckedException {
+ super.insertDataRow(row);
+
+ assert row.key().partition() == PageIdUtils.partId(row.link()) :
+ "Constructed a link with invalid partition ID [partId=" + row.key().partition() +
+ ", link=" + U.hexLong(row.link()) + ']';
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return "FreeList [name=" + name + ']';
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1ecbbc8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
index ca63f27..50b5779 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
@@ -907,6 +907,7 @@ public abstract class AbstractDataPageIO<T extends Storable> extends PageIO {
* Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
*
* @param pageMem Page memory.
+ * @param pageId Page ID to use to construct a link.
* @param pageAddr Page address.
* @param row Data row.
* @param written Number of bytes of row size that was already written.
@@ -917,18 +918,20 @@ public abstract class AbstractDataPageIO<T extends Storable> extends PageIO {
*/
public int addRowFragment(
PageMemory pageMem,
+ long pageId,
long pageAddr,
T row,
int written,
int rowSize,
int pageSize
) throws IgniteCheckedException {
- return addRowFragment(pageMem, pageAddr, written, rowSize, row.link(), row, null, pageSize);
+ return addRowFragment(pageMem, pageId, pageAddr, written, rowSize, row.link(), row, null, pageSize);
}
/**
* Adds this payload as a fragment to this data page.
*
+ * @param pageId Page ID to use to construct a link.
* @param pageAddr Page address.
* @param payload Payload bytes.
* @param lastLink Link to the previous written fragment (link to the tail).
@@ -936,18 +939,20 @@ public abstract class AbstractDataPageIO<T extends Storable> extends PageIO {
* @throws IgniteCheckedException If failed.
*/
public void addRowFragment(
+ long pageId,
long pageAddr,
byte[] payload,
long lastLink,
int pageSize
) throws IgniteCheckedException {
- addRowFragment(null, pageAddr, 0, 0, lastLink, null, payload, pageSize);
+ addRowFragment(null, pageId, pageAddr, 0, 0, lastLink, null, payload, pageSize);
}
/**
* Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
*
* @param pageMem Page memory.
+ * @param pageId Page ID to use to construct a link.
* @param pageAddr Page address.
* @param written Number of bytes of row size that was already written.
* @param rowSize Row size.
@@ -960,6 +965,7 @@ public abstract class AbstractDataPageIO<T extends Storable> extends PageIO {
*/
private int addRowFragment(
PageMemory pageMem,
+ long pageId,
long pageAddr,
int written,
int rowSize,
@@ -1004,24 +1010,13 @@ public abstract class AbstractDataPageIO<T extends Storable> extends PageIO {
int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
if (row != null)
- setLink(row, pageAddr, itemId);
+ setLinkByPageId(row, pageId, itemId);
return payloadSize;
}
/**
* @param row Row to set link to.
- * @param pageAddr Page address.
- * @param itemId Item ID.
- */
- private void setLink(T row, long pageAddr, int itemId) {
- long pageId = getPageId(pageAddr);
-
- setLinkByPageId(row, pageId, itemId);
- }
-
- /**
- * @param row Row to set link to.
* @param pageId Page ID.
* @param itemId Item ID.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1ecbbc8/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BigEntryQueryTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BigEntryQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BigEntryQueryTest.java
new file mode 100644
index 0000000..3b5562a
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BigEntryQueryTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * This is a specific test for IGNITE-8900.
+ */
+public class BigEntryQueryTest extends GridCommonAbstractTest {
+ /** */
+ public static final String CACHE = "cache";
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 60_000;
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testBigEntriesSelect() throws Exception {
+ startGrids(2);
+
+ Random random = new Random(1);
+
+ Ignition.setClientMode(true);
+
+ Ignite client = startGrid(2);
+
+ int ctr = 0;
+
+ long time0 = System.currentTimeMillis();
+
+ while ((System.currentTimeMillis() - time0) < 30_000) {
+ String cacheName = CACHE + ctr++;
+
+ IgniteCache<Long, Value> cache = client.getOrCreateCache(new CacheConfiguration<Long, Value>(cacheName)
+ .setCacheMode(CacheMode.PARTITIONED)
+ .setBackups(1)
+ .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+ .setIndexedTypes(Long.class, Value.class));
+
+ cache.putAll(LongStream.range(610026643276160000L, 610026643276170000L).boxed()
+ .collect(Collectors.toMap(Function.identity(),
+ t -> Value.of(new byte[(random.nextInt(16)) * 1000]))));
+
+ for (int i = 0; i < 10; i++) {
+ long start = 610026643276160000L;
+ long end = start + random.nextInt(10);
+
+ int expectedResultCount = (int)(end - start + 1);
+
+ String sql = String.format(
+ "SELECT _KEY " +
+ "FROM %s " +
+ "WHERE _KEY >= %d AND _KEY <= %d", Value.class.getSimpleName().toLowerCase(),
+ start, end);
+
+ Set<Long> keySet = new HashSet<>();
+ for (long l = start; l < end + 1; l++)
+ keySet.add(l);
+
+ List<Long> resultKeys;
+ try (FieldsQueryCursor<List<?>> results = cache.query(new SqlFieldsQuery(sql))) {
+ resultKeys = new ArrayList<>();
+ results.forEach(objects -> resultKeys.add((Long)objects.get(0)));
+ Collections.sort(resultKeys);
+ }
+
+ assertEquals(expectedResultCount, resultKeys.size());
+ }
+ cache.destroy();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration() throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration();
+
+ cfg.setIncludeEventTypes(EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST);
+
+ DataStorageConfiguration sCfg = new DataStorageConfiguration();
+
+ sCfg.setPageSize(1024 * 8);
+
+ cfg.setDataStorageConfiguration(sCfg);
+
+ return cfg;
+ }
+
+ /**
+ * Class containing value to be placed into the cache.
+ */
+ public static class Value implements Serializable {
+ /** */
+ final byte[] data;
+
+ /**
+ * @param data Data.
+ */
+ public Value(final byte[] data) {
+ this.data = data;
+ }
+
+ /**
+ * @param bytes Bytes.
+ * @return Value.
+ */
+ public static Value of(final byte[] bytes) {
+ return new Value(bytes);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1ecbbc8/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
index 8164fe0..b44ff2d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
@@ -22,6 +22,7 @@ import org.apache.ignite.internal.processors.cache.BinarySerializationQuerySelfT
import org.apache.ignite.internal.processors.cache.BinarySerializationQueryWithReflectiveSerializerSelfTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheBinaryObjectsScanSelfTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheBinaryObjectsScanWithEventsSelfTest;
+import org.apache.ignite.internal.processors.cache.BigEntryQueryTest;
/**
* Cache query suite with binary marshaller.
@@ -39,6 +40,7 @@ public class IgniteBinaryCacheQueryTestSuite extends TestSuite {
suite.addTestSuite(BinarySerializationQueryWithReflectiveSerializerSelfTest.class);
suite.addTestSuite(IgniteCacheBinaryObjectsScanSelfTest.class);
suite.addTestSuite(IgniteCacheBinaryObjectsScanWithEventsSelfTest.class);
+ suite.addTestSuite(BigEntryQueryTest.class);
//Should be adjusted. Not ready to be used with BinaryMarshaller.
//suite.addTestSuite(GridCacheBinarySwapScanQuerySelfTest.class);
[16/50] [abbrv] ignite git commit: IGNITE-8973 Calculate partition
hash and print into standard output
Posted by av...@apache.org.
IGNITE-8973 Calculate partition hash and print into standard output
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8309cef7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8309cef7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8309cef7
Branch: refs/heads/ignite-8446
Commit: 8309cef795e78f9a1a988eb02326278c6f16f1f5
Parents: 5e8669a
Author: Anton Kalashnikov <ka...@yandex.ru>
Authored: Tue Jul 31 16:13:27 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Jul 31 16:13:56 2018 +0300
----------------------------------------------------------------------
.../internal/commandline/CommandHandler.java | 162 ++++++-------
.../commandline/cache/CacheArguments.java | 34 +++
.../cache/verify/IdleVerifyDumpResult.java | 73 ++++++
.../cache/verify/IdleVerifyResultV2.java | 85 +++++--
.../verify/VerifyBackupPartitionsDumpTask.java | 230 +++++++++++++++++++
.../visor/verify/VisorIdleVerifyDumpTask.java | 37 +++
.../verify/VisorIdleVerifyDumpTaskArg.java | 73 ++++++
.../visor/verify/VisorIdleVerifyJob.java | 83 +++++++
.../resources/META-INF/classnames.properties | 2 +
.../ignite/util/GridCommandHandlerTest.java | 178 +++++++++++++-
10 files changed, 843 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8309cef7/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index ca14e08..7b5ce44 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -34,6 +34,7 @@ import java.util.regex.PatternSyntaxException;
import java.util.stream.Collectors;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeTask;
import org.apache.ignite.internal.IgniteNodeAttributes;
import org.apache.ignite.internal.client.GridClient;
import org.apache.ignite.internal.client.GridClientAuthenticationException;
@@ -54,9 +55,7 @@ import org.apache.ignite.internal.processors.cache.verify.CacheInfo;
import org.apache.ignite.internal.processors.cache.verify.ContentionInfo;
import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
-import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2;
import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
-import org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2;
import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.X;
@@ -84,6 +83,8 @@ import org.apache.ignite.internal.visor.verify.ValidateIndexesPartitionResult;
import org.apache.ignite.internal.visor.verify.VisorContentionTask;
import org.apache.ignite.internal.visor.verify.VisorContentionTaskArg;
import org.apache.ignite.internal.visor.verify.VisorContentionTaskResult;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTask;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg;
import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask;
import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult;
@@ -154,8 +155,15 @@ public class CommandHandler {
/** */
protected static final String CMD_PING_TIMEOUT = "--ping-timeout";
+ /** */
+ private static final String CMD_DUMP = "--dump";
+
+ /** */
+ private static final String CMD_SKIP_ZEROS = "--skipZeros";
+
/** List of optional auxiliary commands. */
private static final Set<String> AUX_COMMANDS = new HashSet<>();
+
static {
AUX_COMMANDS.add(CMD_HELP);
AUX_COMMANDS.add(CMD_HOST);
@@ -237,7 +245,7 @@ public class CommandHandler {
private static final String TX_ORDER = "order";
/** */
- public static final String CMD_TX_ORDER_START_TIME="START_TIME";
+ public static final String CMD_TX_ORDER_START_TIME = "START_TIME";
/** */
private static final String TX_SERVERS = "servers";
@@ -463,14 +471,14 @@ public class CommandHandler {
}
/**
- *
* @param client Client.
* @param taskCls Task class.
* @param taskArgs Task arguments.
* @return Task result.
* @throws GridClientException If failed to execute task.
*/
- private <R> R executeTask(GridClient client, Class<?> taskCls, Object taskArgs) throws GridClientException {
+ private <R> R executeTask(GridClient client, Class<? extends ComputeTask<?, R>> taskCls,
+ Object taskArgs) throws GridClientException {
return executeTaskByNameOnNode(client, taskCls.getName(), taskArgs, null);
}
@@ -595,7 +603,7 @@ public class CommandHandler {
usage(" Show information about caches, groups or sequences that match a regex:", CACHE, " list regexPattern [groups|seq] [nodeId]");
usage(" Show hot keys that are point of contention for multiple transactions:", CACHE, " contention minQueueSize [nodeId] [maxPrint]");
- usage(" Verify partition counters and hashes between primary and backups on idle cluster:", CACHE, " idle_verify [cache1,...,cacheN]");
+ usage(" Verify partition counters and hashes between primary and backups on idle cluster:", CACHE, " idle_verify [--dump] [--skipZeros] [cache1,...,cacheN]");
usage(" Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId] [checkFirst|checkThrough]");
log(" If [nodeId] is not specified, contention and validate_indexes commands will be broadcasted to all server nodes.");
@@ -717,7 +725,8 @@ public class CommandHandler {
}
/**
- * Executes appropriate version of idle_verify check. Old version will be used if there are old nodes in the cluster.
+ * Executes appropriate version of idle_verify check. Old version will be used if there are old nodes in the
+ * cluster.
*
* @param client Client.
* @param cacheArgs Cache args.
@@ -739,7 +748,9 @@ public class CommandHandler {
}
}
- if (idleVerifyV2)
+ if (cacheArgs.dump())
+ cacheIdleVerifyDump(client, cacheArgs);
+ else if (idleVerifyV2)
cacheIdleVerifyV2(client, cacheArgs);
else
legacyCacheIdleVerify(client, cacheArgs);
@@ -760,7 +771,7 @@ public class CommandHandler {
nl();
}
else {
- log ("idle_verify check has finished, found " + conflicts.size() + " conflict partitions.");
+ log("idle_verify check has finished, found " + conflicts.size() + " conflict partitions.");
nl();
for (Map.Entry<PartitionKey, List<PartitionHashRecord>> entry : conflicts.entrySet()) {
@@ -775,66 +786,33 @@ public class CommandHandler {
* @param client Client.
* @param cacheArgs Cache args.
*/
+ private void cacheIdleVerifyDump(GridClient client, CacheArguments cacheArgs) throws GridClientException {
+ String path = executeTask(
+ client,
+ VisorIdleVerifyDumpTask.class,
+ new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.isSkipZeros())
+ );
+
+ log("VisorIdleVerifyDumpTask successfully written output to '" + path + "'");
+ }
+
+ /**
+ * @param client Client.
+ * @param cacheArgs Cache args.
+ */
private void cacheIdleVerifyV2(GridClient client, CacheArguments cacheArgs) throws GridClientException {
IdleVerifyResultV2 res = executeTask(
client, VisorIdleVerifyTaskV2.class, new VisorIdleVerifyTaskArg(cacheArgs.caches()));
- if (!res.hasConflicts()) {
- log("idle_verify check has finished, no conflicts have been found.");
- nl();
- }
- else {
- int cntrConflictsSize = res.counterConflicts().size();
- int hashConflictsSize = res.hashConflicts().size();
-
- log("idle_verify check has finished, found " + (cntrConflictsSize + hashConflictsSize) +
- " conflict partitions: [counterConflicts=" + cntrConflictsSize + ", hashConflicts=" +
- hashConflictsSize + "]");
- nl();
-
- if (!F.isEmpty(res.counterConflicts())) {
- log("Update counter conflicts:");
-
- for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : res.counterConflicts().entrySet()) {
- log("Conflict partition: " + entry.getKey());
-
- log("Partition instances: " + entry.getValue());
- }
-
- nl();
- }
-
- if (!F.isEmpty(res.hashConflicts())) {
- log("Hash conflicts:");
-
- for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : res.hashConflicts().entrySet()) {
- log("Conflict partition: " + entry.getKey());
-
- log("Partition instances: " + entry.getValue());
- }
-
- nl();
- }
- }
-
- if (!F.isEmpty(res.movingPartitions())) {
- log("Verification was skipped for " + res.movingPartitions().size() + " MOVING partitions:");
-
- for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : res.movingPartitions().entrySet()) {
- log("Rebalancing partition: " + entry.getKey());
-
- log("Partition instances: " + entry.getValue());
- }
-
- nl();
- }
+ res.print(System.out::print);
}
/**
* Change baseline.
*
* @param client Client.
- * @param baselineAct Baseline action to execute. @throws GridClientException If failed to execute baseline action.
+ * @param baselineAct Baseline action to execute. @throws GridClientException If failed to execute baseline
+ * action.
* @param baselineArgs Baseline action arguments.
* @throws Throwable If failed to execute baseline action.
*/
@@ -928,7 +906,7 @@ public class CommandHandler {
else {
log("Baseline nodes:");
- for(VisorBaselineNode node : baseline.values()) {
+ for (VisorBaselineNode node : baseline.values()) {
log(" ConsistentID=" + node.getConsistentId() + ", STATE=" +
(srvs.containsKey(node.getConsistentId()) ? "ONLINE" : "OFFLINE"));
}
@@ -950,7 +928,7 @@ public class CommandHandler {
else {
log("Other nodes:");
- for(VisorBaselineNode node : others)
+ for (VisorBaselineNode node : others)
log(" ConsistentID=" + node.getConsistentId());
log("Number of other nodes: " + others.size());
@@ -1123,7 +1101,7 @@ public class CommandHandler {
* @throws Throwable If failed to execute wal action.
*/
private void wal(GridClient client, String walAct, String walArgs) throws Throwable {
- switch (walAct){
+ switch (walAct) {
case WAL_DELETE:
deleteUnusedWalSegments(client, walArgs);
@@ -1145,7 +1123,7 @@ public class CommandHandler {
*/
private void deleteUnusedWalSegments(GridClient client, String walArgs) throws Throwable {
VisorWalTaskResult res = executeTask(client, VisorWalTask.class,
- walArg(VisorWalTaskOperation.DELETE_UNUSED_WAL_SEGMENTS, walArgs));
+ walArg(VisorWalTaskOperation.DELETE_UNUSED_WAL_SEGMENTS, walArgs));
printDeleteWalSegments0(res);
}
@@ -1157,7 +1135,7 @@ public class CommandHandler {
*/
private void printUnusedWalSegments(GridClient client, String walArgs) throws Throwable {
VisorWalTaskResult res = executeTask(client, VisorWalTask.class,
- walArg(VisorWalTaskOperation.PRINT_UNUSED_WAL_SEGMENTS, walArgs));
+ walArg(VisorWalTaskOperation.PRINT_UNUSED_WAL_SEGMENTS, walArgs));
printUnusedWalSegments0(res);
}
@@ -1168,7 +1146,7 @@ public class CommandHandler {
* @param s Argument from command line.
* @return Task argument.
*/
- private VisorWalTaskArg walArg(VisorWalTaskOperation op, String s){
+ private VisorWalTaskArg walArg(VisorWalTaskOperation op, String s) {
List<String> consistentIds = null;
if (!F.isEmpty(s)) {
@@ -1202,23 +1180,23 @@ public class CommandHandler {
Map<String, Exception> failRes = taskRes.exceptions();
Map<String, VisorClusterNode> nodesInfo = taskRes.getNodesInfo();
- for(Map.Entry<String, Collection<String>> entry: res.entrySet()) {
+ for (Map.Entry<String, Collection<String>> entry : res.entrySet()) {
VisorClusterNode node = nodesInfo.get(entry.getKey());
log("Node=" + node.getConsistentId());
- log(" addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+ log(" addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()));
- for(String fileName: entry.getValue())
+ for (String fileName : entry.getValue())
log(" " + fileName);
nl();
}
- for(Map.Entry<String, Exception> entry: failRes.entrySet()) {
+ for (Map.Entry<String, Exception> entry : failRes.entrySet()) {
VisorClusterNode node = nodesInfo.get(entry.getKey());
log("Node=" + node.getConsistentId());
- log(" addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+ log(" addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()));
log(" failed with error: " + entry.getValue().getMessage());
nl();
}
@@ -1237,19 +1215,19 @@ public class CommandHandler {
Map<String, Exception> errors = taskRes.exceptions();
Map<String, VisorClusterNode> nodesInfo = taskRes.getNodesInfo();
- for(Map.Entry<String, Collection<String>> entry: res.entrySet()) {
+ for (Map.Entry<String, Collection<String>> entry : res.entrySet()) {
VisorClusterNode node = nodesInfo.get(entry.getKey());
log("Node=" + node.getConsistentId());
- log(" addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+ log(" addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()));
nl();
}
- for(Map.Entry<String, Exception> entry: errors.entrySet()) {
+ for (Map.Entry<String, Exception> entry : errors.entrySet()) {
VisorClusterNode node = nodesInfo.get(entry.getKey());
log("Node=" + node.getConsistentId());
- log(" addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+ log(" addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()));
log(" failed with error: " + entry.getValue().getMessage());
nl();
}
@@ -1284,7 +1262,7 @@ public class CommandHandler {
private void usage(String desc, Command cmd, String... args) {
log(desc);
log(" control.sh [--host HOST_OR_IP] [--port PORT] [--user USER] [--password PASSWORD] " +
- " [--ping-interval PING_INTERVAL] [--ping-timeout PING_TIMEOUT] " + cmd.text() + String.join("", args));
+ " [--ping-interval PING_INTERVAL] [--ping-timeout PING_TIMEOUT] " + cmd.text() + String.join("", args));
nl();
}
@@ -1419,8 +1397,8 @@ public class CommandHandler {
if (WAL_PRINT.equals(walAct) || WAL_DELETE.equals(walAct))
walArgs = (str = peekNextArg()) != null && !isCommandOrOption(str)
- ? nextArg("Unexpected argument for " + WAL.text() + ": " + walAct)
- : "";
+ ? nextArg("Unexpected argument for " + WAL.text() + ": " + walAct)
+ : "";
else
throw new IllegalArgumentException("Unexpected action " + walAct + " for " + WAL.text());
@@ -1499,7 +1477,7 @@ public class CommandHandler {
throw new IllegalArgumentException("Both user and password should be specified");
return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs,
- pingTimeout, pingInterval, autoConfirmation);
+ pingTimeout, pingInterval, autoConfirmation);
}
/**
@@ -1529,9 +1507,18 @@ public class CommandHandler {
break;
case IDLE_VERIFY:
- if (hasNextCacheArg())
- parseCacheNames(nextArg(""), cacheArgs);
+ int idleVerifyArgsCnt = 3;
+
+ while (hasNextCacheArg() && idleVerifyArgsCnt-- > 0) {
+ String nextArg = nextArg("");
+ if (CMD_DUMP.equals(nextArg))
+ cacheArgs.dump(true);
+ else if (CMD_SKIP_ZEROS.equals(nextArg))
+ cacheArgs.skipZeros(true);
+ else
+ parseCacheNames(nextArg, cacheArgs);
+ }
break;
case CONTENTION:
@@ -1709,7 +1696,7 @@ public class CommandHandler {
case TX_LIMIT:
nextArg("");
- limit = (int) nextLongArg(TX_LIMIT);
+ limit = (int)nextLongArg(TX_LIMIT);
break;
case TX_ORDER:
@@ -1746,7 +1733,7 @@ public class CommandHandler {
case TX_SIZE:
nextArg("");
- size = (int) nextLongArg(TX_SIZE);
+ size = (int)nextLongArg(TX_SIZE);
break;
case TX_LABEL:
@@ -1807,9 +1794,9 @@ public class CommandHandler {
}
/**
- * Check if raw arg is command or option.
+ * Check if raw arg is command or option.
*
- * @return {@code true} If raw arg is command, overwise {@code false}.
+ * @return {@code true} If raw arg is command, overwise {@code false}.
*/
private boolean isCommandOrOption(String raw) {
return raw != null && raw.contains("--");
@@ -1843,11 +1830,11 @@ public class CommandHandler {
"[minSize SIZE] [label PATTERN_REGEX] [servers|clients] " +
"[nodes consistentId1[,consistentId2,....,consistentIdN] [limit NUMBER] [order DURATION|SIZE|", CMD_TX_ORDER_START_TIME, "] [kill] [" + CMD_AUTO_CONFIRMATION + "]");
- if(enableExperimental) {
+ if (enableExperimental) {
usage(" Print absolute paths of unused archived wal segments on each node:", WAL,
- " print [consistentId1,consistentId2,....,consistentIdN]");
+ " print [consistentId1,consistentId2,....,consistentIdN]");
usage(" Delete unused archived wal segments on each node:", WAL,
- " delete [consistentId1,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]");
+ " delete [consistentId1,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]");
}
log(" View caches information in a cluster. For more details type:");
@@ -1962,6 +1949,7 @@ public class CommandHandler {
/**
* Used for tests.
+ *
* @return Last operation result;
*/
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/ignite/blob/8309cef7/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
index 1411b2a..353a63c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
@@ -55,6 +55,12 @@ public class CacheArguments {
/** Cache view command. */
private @Nullable VisorViewCacheCmd cacheCmd;
+ /** Calculate partition hash and print into standard output. */
+ private boolean dump;
+
+ /** Skip zeros partitions. */
+ private boolean skipZeros;
+
/**
* @return Command.
*/
@@ -194,4 +200,32 @@ public class CacheArguments {
public void checkThrough(int checkThrough) {
this.checkThrough = checkThrough;
}
+
+ /**
+ * @return Calculate partition hash and print into standard output.
+ */
+ public boolean dump() {
+ return dump;
+ }
+
+ /**
+ * @param dump Calculate partition hash and print into standard output.
+ */
+ public void dump(boolean dump) {
+ this.dump = dump;
+ }
+
+ /**
+ * @return Skip zeros partitions(size == 0) in result.
+ */
+ public boolean isSkipZeros() {
+ return skipZeros;
+ }
+
+ /**
+ * @param skipZeros Skip zeros partitions.
+ */
+ public void skipZeros(boolean skipZeros) {
+ this.skipZeros = skipZeros;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8309cef7/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyDumpResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyDumpResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyDumpResult.java
new file mode 100644
index 0000000..6a74405
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyDumpResult.java
@@ -0,0 +1,73 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Encapsulates result of {@link VerifyBackupPartitionsDumpTask}.
+ */
+public class IdleVerifyDumpResult extends VisorDataTransferObject {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Cluster hashes. */
+ private Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes;
+
+ /**
+ * @param clusterHashes Cluster hashes.
+ */
+ public IdleVerifyDumpResult(Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes) {
+ this.clusterHashes = clusterHashes;
+ }
+
+ /**
+ * Default constructor for Externalizable.
+ */
+ public IdleVerifyDumpResult() {
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+ U.writeMap(out, clusterHashes);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void readExternalData(byte protoVer,
+ ObjectInput in) throws IOException, ClassNotFoundException {
+ clusterHashes = U.readLinkedMap(in);
+ }
+
+ /**
+ * @return Cluster hashes.
+ */
+ public Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes() {
+ return clusterHashes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IdleVerifyDumpResult.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8309cef7/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
index d5815cd..c31d662 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
@@ -1,19 +1,19 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.ignite.internal.processors.cache.verify;
import java.io.IOException;
@@ -21,6 +21,7 @@ import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.List;
import java.util.Map;
+import java.util.function.Consumer;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -106,6 +107,60 @@ public class IdleVerifyResultV2 extends VisorDataTransferObject {
return !F.isEmpty(hashConflicts()) || !F.isEmpty(counterConflicts());
}
+ /**
+ * Print formatted result to given printer.
+ *
+ * @param printer Consumer for handle formatted result.
+ */
+ public void print(Consumer<String> printer) {
+ if (!hasConflicts())
+ printer.accept("idle_verify check has finished, no conflicts have been found.\n");
+ else {
+ int cntrConflictsSize = counterConflicts().size();
+ int hashConflictsSize = hashConflicts().size();
+
+ printer.accept("idle_verify check has finished, found " + (cntrConflictsSize + hashConflictsSize) +
+ " conflict partitions: [counterConflicts=" + cntrConflictsSize + ", hashConflicts=" +
+ hashConflictsSize + "]\n");
+
+ if (!F.isEmpty(counterConflicts())) {
+ printer.accept("Update counter conflicts:\n");
+
+ for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : counterConflicts().entrySet()) {
+ printer.accept("Conflict partition: " + entry.getKey() + "\n");
+
+ printer.accept("Partition instances: " + entry.getValue() + "\n");
+ }
+
+ printer.accept("\n");
+ }
+
+ if (!F.isEmpty(hashConflicts())) {
+ printer.accept("Hash conflicts:\n");
+
+ for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : hashConflicts().entrySet()) {
+ printer.accept("Conflict partition: " + entry.getKey() + "\n");
+
+ printer.accept("Partition instances: " + entry.getValue() + "\n");
+ }
+
+ printer.accept("\n");
+ }
+ }
+
+ if (!F.isEmpty(movingPartitions())) {
+ printer.accept("Verification was skipped for " + movingPartitions().size() + " MOVING partitions:\n");
+
+ for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : movingPartitions().entrySet()) {
+ printer.accept("Rebalancing partition: " + entry.getKey() + "\n");
+
+ printer.accept("Partition instances: " + entry.getValue() + "\n");
+ }
+
+ printer.accept("\n");
+ }
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(IdleVerifyResultV2.class, this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/8309cef7/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
new file mode 100644
index 0000000..7bbd9f1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeTaskAdapter;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task for collection checksums primary and backup partitions of specified caches. <br> Argument: Set of cache names,
+ * 'null' will trigger verification for all caches. <br> Result: {@link IdleVerifyDumpResult} with all found partitions.
+ * <br> Works properly only on idle cluster - there may be false positive conflict reports if data in cluster is being
+ * concurrently updated.
+ */
+@GridInternal
+public class VerifyBackupPartitionsDumpTask extends ComputeTaskAdapter<VisorIdleVerifyTaskArg, String> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Time formatter for dump file name. */
+ private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH-mm-ss_SSS");
+
+ /** Visible for testing. */
+ public static final String IDLE_DUMP_FILE_PREMIX = "idle-dump-";
+
+ /** Delegate for map execution */
+ private final VerifyBackupPartitionsTaskV2 delegate = new VerifyBackupPartitionsTaskV2();
+
+ /** */
+ private VisorIdleVerifyDumpTaskArg taskArg;
+
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(
+ List<ClusterNode> subgrid, VisorIdleVerifyTaskArg arg) throws IgniteException {
+ if (arg instanceof VisorIdleVerifyDumpTaskArg)
+ taskArg = (VisorIdleVerifyDumpTaskArg)arg;
+
+ return delegate.map(subgrid, arg);
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public String reduce(List<ComputeJobResult> results)
+ throws IgniteException {
+ Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes = new TreeMap<>(buildPartitionKeyComparator());
+
+ for (ComputeJobResult res : results) {
+ Map<PartitionKeyV2, PartitionHashRecordV2> nodeHashes = res.getData();
+
+ for (Map.Entry<PartitionKeyV2, PartitionHashRecordV2> e : nodeHashes.entrySet()) {
+ clusterHashes
+ .computeIfAbsent(e.getKey(), k -> new ArrayList<>())
+ .add(e.getValue());
+ }
+ }
+
+ Comparator<PartitionHashRecordV2> recordComp = buildRecordComparator().reversed();
+
+ Map<PartitionKeyV2, List<PartitionHashRecordV2>> partitions = new LinkedHashMap<>();
+
+ int skippedRecords = 0;
+
+ for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : clusterHashes.entrySet()) {
+ if (needToAdd(entry.getValue())) {
+ entry.getValue().sort(recordComp);
+
+ partitions.put(entry.getKey(), entry.getValue());
+ }
+ else
+ skippedRecords++;
+ }
+
+ return writeHashes(partitions, delegate.reduce(results), skippedRecords);
+ }
+
+ /**
+ * Checking conditions for adding given record to result.
+ *
+ * @param records records to check.
+ * @return {@code true} if this records should be add to result and {@code false} otherwise.
+ */
+ private boolean needToAdd(List<PartitionHashRecordV2> records) {
+ if (records.isEmpty() || (taskArg != null && !taskArg.isSkipZeros()))
+ return true;
+
+ PartitionHashRecordV2 record = records.get(0);
+
+ if (record.updateCounter() != 0 || record.size() != 0)
+ return true;
+
+ int firstHash = record.partitionHash();
+
+ for (int i = 1; i < records.size(); i++) {
+ record = records.get(i);
+
+ if (record.partitionHash() != firstHash || record.updateCounter() != 0 || record.size() != 0)
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @param partitions Dump result.
+ * @return Path where results are written.
+ * @throws IgniteException If failed to write the file.
+ */
+ private String writeHashes(
+ Map<PartitionKeyV2, List<PartitionHashRecordV2>> partitions,
+ IdleVerifyResultV2 conflictRes,
+ int skippedRecords
+ ) throws IgniteException {
+ File workDir = ignite.configuration().getWorkDirectory() == null
+ ? new File("/tmp")
+ : new File(ignite.configuration().getWorkDirectory());
+
+ File out = new File(workDir, IDLE_DUMP_FILE_PREMIX + LocalDateTime.now().format(TIME_FORMATTER) + ".txt");
+
+ ignite.log().info("IdleVerifyDumpTask will write output to " + out.getAbsolutePath());
+
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(out))) {
+ try {
+
+ writer.write("idle_verify check has finished, found " + partitions.size() + " partitions\n");
+
+ if (skippedRecords > 0)
+ writer.write(skippedRecords + " partitions was skipped\n");
+
+ if (!F.isEmpty(partitions)) {
+ writer.write("Cluster partitions:\n");
+
+ for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : partitions.entrySet()) {
+ writer.write("Partition: " + entry.getKey() + "\n");
+
+ writer.write("Partition instances: " + entry.getValue() + "\n");
+ }
+
+ writer.write("\n\n-----------------------------------\n\n");
+
+ conflictRes.print(str -> {
+ try {
+ writer.write(str);
+ }
+ catch (IOException e) {
+ throw new IgniteException("Failed to write partitions conflict.", e);
+ }
+ });
+ }
+ }
+ finally {
+ writer.flush();
+ }
+
+ ignite.log().info("IdleVerifyDumpTask successfully written dump to '" + out.getAbsolutePath() + "'");
+ }
+ catch (IOException | IgniteException e) {
+ ignite.log().error("Failed to write dump file: " + out.getAbsolutePath(), e);
+
+ throw new IgniteException(e);
+ }
+
+ return out.getAbsolutePath();
+ }
+
+ /**
+ * @return Comparator for {@link PartitionHashRecordV2}.
+ */
+ @NotNull private Comparator<PartitionHashRecordV2> buildRecordComparator() {
+ return (o1, o2) -> {
+ int compare = Boolean.compare(o1.isPrimary(), o2.isPrimary());
+
+ if (compare != 0)
+ return compare;
+
+ return o1.consistentId().toString().compareTo(o2.consistentId().toString());
+ };
+ }
+
+ /**
+ * @return Comparator for {@link PartitionKeyV2}.
+ */
+ @NotNull private Comparator<PartitionKeyV2> buildPartitionKeyComparator() {
+ return (o1, o2) -> {
+ int compare = Integer.compare(o1.groupId(), o2.groupId());
+
+ if (compare != 0)
+ return compare;
+
+ return Integer.compare(o1.partitionId(), o2.partitionId());
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8309cef7/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTask.java
new file mode 100644
index 0000000..8f34c68
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTask.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsDumpTask;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+
+/**
+ * Task to verify checksums of backup partitions and return all collected information.
+ */
+@GridInternal
+public class VisorIdleVerifyDumpTask extends VisorOneNodeTask<VisorIdleVerifyTaskArg, String> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override protected VisorJob<VisorIdleVerifyTaskArg, String> job(VisorIdleVerifyTaskArg arg) {
+ return new VisorIdleVerifyJob<>(arg, debug, VerifyBackupPartitionsDumpTask.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8309cef7/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
new file mode 100644
index 0000000..6316c24
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Set;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Arguments for {@link VisorIdleVerifyDumpTask}.
+ */
+public class VisorIdleVerifyDumpTaskArg extends VisorIdleVerifyTaskArg {
+ /** */
+ private static final long serialVersionUID = 0L;
+ /** */
+ private boolean skipZeros;
+
+ /**
+ * Default constructor.
+ */
+ public VisorIdleVerifyDumpTaskArg() {
+ }
+
+ /**
+ * @param caches Caches.
+ * @param skipZeros Skip zeros partitions.
+ */
+ public VisorIdleVerifyDumpTaskArg(Set<String> caches, boolean skipZeros) {
+ super(caches);
+ this.skipZeros = skipZeros;
+ }
+
+ /**
+ * @return Skip zeros partitions.
+ */
+ public boolean isSkipZeros() {
+ return skipZeros;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+ super.writeExternalData(out);
+ out.writeBoolean(skipZeros);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+ super.readExternalData(protoVer, in);
+ skipZeros = in.readBoolean();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(VisorIdleVerifyDumpTaskArg.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8309cef7/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyJob.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyJob.java
new file mode 100644
index 0000000..a8dc697
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyJob.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobContext;
+import org.apache.ignite.compute.ComputeTask;
+import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.resources.JobContextResource;
+
+/**
+ *
+ */
+class VisorIdleVerifyJob<ResultT> extends VisorJob<VisorIdleVerifyTaskArg, ResultT> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private ComputeTaskFuture<ResultT> fut;
+
+ /** Auto-inject job context. */
+ @JobContextResource
+ protected transient ComputeJobContext jobCtx;
+
+ /** Task class for execution */
+ private final Class<? extends ComputeTask<VisorIdleVerifyTaskArg, ResultT>> taskCls;
+
+ /**
+ * @param arg Argument.
+ * @param debug Debug.
+ * @param taskCls Task class for execution.
+ */
+ VisorIdleVerifyJob(VisorIdleVerifyTaskArg arg, boolean debug,
+ Class<? extends ComputeTask<VisorIdleVerifyTaskArg, ResultT>> taskCls) {
+ super(arg, debug);
+ this.taskCls = taskCls;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected ResultT run(VisorIdleVerifyTaskArg arg) throws IgniteException {
+ if (fut == null) {
+ fut = ignite.compute().executeAsync(taskCls, arg);
+
+ if (!fut.isDone()) {
+ jobCtx.holdcc();
+
+ fut.listen(new IgniteInClosure<IgniteFuture<ResultT>>() {
+ @Override public void apply(IgniteFuture<ResultT> f) {
+ jobCtx.callcc();
+ }
+ });
+
+ return null;
+ }
+ }
+
+ return fut.get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(VisorIdleVerifyJob.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8309cef7/modules/core/src/main/resources/META-INF/classnames.properties
----------------------------------------------------------------------
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 02e2e78..d4bbf0b 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -1138,6 +1138,7 @@ org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysT
org.apache.ignite.internal.processors.cache.verify.ContentionClosure
org.apache.ignite.internal.processors.cache.verify.ContentionInfo
org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2
+org.apache.ignite.internal.processors.cache.verify.IdleVerifyDumpResult
org.apache.ignite.internal.processors.cache.verify.PartitionEntryHashRecord
org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord
org.apache.ignite.internal.processors.cache.verify.PartitionKey
@@ -2159,6 +2160,7 @@ org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask
org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask$VisorIdleVerifyJob
org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask$VisorIdleVerifyJob$1
org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg
org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult
org.apache.ignite.internal.visor.verify.VisorValidateIndexesJobResult
org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskArg
http://git-wip-us.apache.org/repos/asf/ignite/blob/8309cef7/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index c37b17c..f612a96 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -17,9 +17,16 @@
package org.apache.ignite.util;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -33,9 +40,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.LongAdder;
-import javax.cache.processor.EntryProcessor;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.MutableEntry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteAtomicSequence;
import org.apache.ignite.IgniteCache;
@@ -59,18 +65,17 @@ import org.apache.ignite.internal.processors.cache.CacheObjectImpl;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
import org.apache.ignite.internal.processors.cache.GridCacheFuture;
-import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
+import org.apache.ignite.internal.processors.cache.GridCacheOperation;
+import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRequest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockResponse;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl;
-import org.apache.ignite.internal.processors.cache.GridCacheOperation;
-import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.typedef.F;
@@ -89,12 +94,16 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
import org.apache.ignite.transactions.TransactionRollbackException;
import org.apache.ignite.transactions.TransactionTimeoutException;
+import org.jetbrains.annotations.NotNull;
+import static java.nio.file.Files.delete;
+import static java.nio.file.Files.newDirectoryStream;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR;
+import static org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsDumpTask.IDLE_DUMP_FILE_PREMIX;
import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
@@ -139,6 +148,15 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
cleanPersistenceDir();
+ //delete idle-verify dump files.
+ try (DirectoryStream<Path> files = newDirectoryStream(
+ Paths.get(U.defaultWorkDirectory()),
+ entry -> entry.toFile().getName().startsWith(IDLE_DUMP_FILE_PREMIX)
+ )) {
+ for (Path path : files)
+ delete(path);
+ }
+
System.clearProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND);
System.setOut(sysOut);
@@ -505,14 +523,14 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
validate(h, map -> {
VisorTxTaskResult res = map.get(grid(0).localNode());
- assertTrue(res.getInfos().get(0).getSize() >= res.getInfos().get(1).getSize());
+ assertTrue(res.getInfos().get(0).getSize() >= res.getInfos().get(1).getSize());
}, "--tx", "order", "SIZE");
// test order by duration.
validate(h, map -> {
VisorTxTaskResult res = map.get(grid(0).localNode());
- assertTrue(res.getInfos().get(0).getDuration() >= res.getInfos().get(1).getDuration());
+ assertTrue(res.getInfos().get(0).getDuration() >= res.getInfos().get(1).getDuration());
}, "--tx", "order", "DURATION");
// test order by start_time.
@@ -585,7 +603,7 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
GridNearTxLocal clientTx = null;
- try(Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 2000, 1)) {
+ try (Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 2000, 1)) {
clientTx = ((TransactionProxyImpl)tx).tx();
client.cache(DEFAULT_CACHE_NAME).put(0L, 0L);
@@ -716,7 +734,7 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
primSpi.waitForBlocked(clients.length);
// Unblock only finish messages from clients from 2 to 4.
- primSpi.stopBlock(true, new IgnitePredicate<T2<ClusterNode,GridIoMessage>>() {
+ primSpi.stopBlock(true, new IgnitePredicate<T2<ClusterNode, GridIoMessage>>() {
@Override public boolean apply(T2<ClusterNode, GridIoMessage> objects) {
GridIoMessage iom = objects.get2();
@@ -921,6 +939,141 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
}
/**
+ * Tests that idle verify print partitions info.
+ *
+ * @throws Exception If failed.
+ */
+ public void testCacheIdleVerifyDump() throws Exception {
+ IgniteEx ignite = (IgniteEx)startGrids(3);
+
+ ignite.cluster().active(true);
+
+ int parts = 32;
+
+ IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME));
+
+ ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME + "other"));
+
+ injectTestSystemOut();
+
+ int keysCount = 20;//less than parts number for ability to check skipZeros flag.
+
+ for (int i = 0; i < keysCount; i++)
+ cache.put(i, i);
+
+ assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", DEFAULT_CACHE_NAME));
+
+ assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--skipZeros", DEFAULT_CACHE_NAME));
+
+ Matcher fileNameMatcher = dumpFileNameMatcher();
+
+ if (fileNameMatcher.find()) {
+ String dumpWithZeros = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+ assertTrue(dumpWithZeros.contains("idle_verify check has finished, found " + parts + " partitions"));
+ assertTrue(dumpWithZeros.contains("Partition: PartitionKeyV2 [grpId=1544803905, grpName=default, partId=0]"));
+ assertTrue(dumpWithZeros.contains("updateCntr=0, size=0, partHash=0"));
+ assertTrue(dumpWithZeros.contains("no conflicts have been found"));
+
+ assertSort(parts, dumpWithZeros);
+ }
+
+ if (fileNameMatcher.find()) {
+ String dumpWithoutZeros = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+ assertTrue(dumpWithoutZeros.contains("idle_verify check has finished, found " + keysCount + " partitions"));
+ assertTrue(dumpWithoutZeros.contains((parts - keysCount) + " partitions was skipped"));
+ assertTrue(dumpWithoutZeros.contains("Partition: PartitionKeyV2 [grpId=1544803905, grpName=default, partId="));
+
+ assertFalse(dumpWithoutZeros.contains("updateCntr=0, size=0, partHash=0"));
+
+ assertTrue(dumpWithoutZeros.contains("no conflicts have been found"));
+
+ assertSort(keysCount, dumpWithoutZeros);
+ }
+ else
+ fail("Should be found both files");
+ }
+
+ /**
+ * Checking sorting of partitions.
+ *
+ * @param expectedPartsCount Expected parts count.
+ * @param output Output.
+ */
+ private void assertSort(int expectedPartsCount, String output) {
+ Pattern partIdPattern = Pattern.compile(".*partId=([0-9]*)");
+ Pattern primaryPattern = Pattern.compile("Partition instances: \\[PartitionHashRecordV2 \\[isPrimary=true");
+
+ Matcher partIdMatcher = partIdPattern.matcher(output);
+ Matcher primaryMatcher = primaryPattern.matcher(output);
+
+ int i = 0;
+
+ while (partIdMatcher.find()) {
+ assertEquals(i++, Integer.parseInt(partIdMatcher.group(1)));
+ assertTrue(primaryMatcher.find());//primary node should be first in every line
+ }
+
+ assertEquals(expectedPartsCount, i);
+ }
+
+ /**
+ * Tests that idle verify print partitions info.
+ *
+ * @throws Exception If failed.
+ */
+ public void testCacheIdleVerifyDumpForCorruptedData() throws Exception {
+ IgniteEx ignite = (IgniteEx)startGrids(3);
+
+ ignite.cluster().active(true);
+
+ int parts = 32;
+
+ IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME));
+
+ injectTestSystemOut();
+
+ for (int i = 0; i < 100; i++)
+ cache.put(i, i);
+
+ GridCacheContext<Object, Object> cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context();
+
+ corruptDataEntry(cacheCtx, 0, true, false);
+
+ corruptDataEntry(cacheCtx, 0 + parts / 2, false, true);
+
+ assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump"));
+
+ Matcher fileNameMatcher = dumpFileNameMatcher();
+
+ if (fileNameMatcher.find()) {
+ String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+ assertTrue(dumpWithConflicts.contains("found 2 conflict partitions: [counterConflicts=1, hashConflicts=1]"));
+ }else
+ fail("Should be found dump with conflicts");
+ }
+
+ /**
+ * @return Build matcher for dump file name.
+ */
+ @NotNull private Matcher dumpFileNameMatcher() {
+ Pattern fileNamePattern = Pattern.compile(".*VisorIdleVerifyDumpTask successfully written output to '(.*)'");
+
+ return fileNamePattern.matcher(testOut.toString());
+ }
+
+ /**
* @throws Exception If failed.
*/
public void testCacheIdleVerifyMovingParts() throws Exception {
@@ -1288,7 +1441,8 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
/** */
private static class IncrementClosure implements EntryProcessor<Long, Long, Void> {
/** {@inheritDoc} */
- @Override public Void process(MutableEntry<Long, Long> entry, Object... arguments) throws EntryProcessorException {
+ @Override public Void process(MutableEntry<Long, Long> entry,
+ Object... arguments) throws EntryProcessorException {
entry.setValue(entry.exists() ? entry.getValue() + 1 : 0);
return null;
[06/50] [abbrv] ignite git commit: IGNITE-9105 Web Console: Added
support of canvas based charts.
Posted by av...@apache.org.
IGNITE-9105 Web Console: Added support of canvas based charts.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4d9f4504
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4d9f4504
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4d9f4504
Branch: refs/heads/ignite-8446
Commit: 4d9f45041d8cb1ece1e2612d265fa3b792246857
Parents: 1b5d275
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Tue Jul 31 10:24:32 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Jul 31 10:24:32 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/app/app.js | 4 +
.../ignite-chart-series-selector/component.js | 28 ++
.../ignite-chart-series-selector/controller.js | 62 ++++
.../ignite-chart-series-selector/index.js | 24 ++
.../ignite-chart-series-selector/template.pug | 29 ++
.../app/components/ignite-chart/controller.js | 320 +++++++++++++++++++
.../app/components/ignite-chart/index.js | 38 +++
.../app/components/ignite-chart/style.scss | 69 ++++
.../app/components/ignite-chart/template.pug | 36 +++
.../frontend/app/primitives/btn/index.scss | 9 +
modules/web-console/frontend/package.json | 2 +
11 files changed, 621 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4d9f4504/modules/web-console/frontend/app/app.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index baf06dd..e807319 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -142,6 +142,8 @@ import pageLanding from './components/page-landing';
import passwordVisibility from './components/password-visibility';
import progressLine from './components/progress-line';
import formField from './components/form-field';
+import igniteChart from './components/ignite-chart';
+import igniteChartSelector from './components/ignite-chart-series-selector';
import pageProfile from './components/page-profile';
import pagePasswordChanged from './components/page-password-changed';
@@ -249,6 +251,8 @@ export default angular.module('ignite-console', [
uiAceSpring.name,
breadcrumbs.name,
passwordVisibility.name,
+ igniteChart.name,
+ igniteChartSelector.name,
progressLine.name,
formField.name
])
http://git-wip-us.apache.org/repos/asf/ignite/blob/4d9f4504/modules/web-console/frontend/app/components/ignite-chart-series-selector/component.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ignite-chart-series-selector/component.js b/modules/web-console/frontend/app/components/ignite-chart-series-selector/component.js
new file mode 100644
index 0000000..41549fc
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-chart-series-selector/component.js
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import template from './template.pug';
+import controller from './controller.js';
+
+export default {
+ template,
+ controller,
+ transclude: true,
+ bindings: {
+ chartApi: '<'
+ }
+};
http://git-wip-us.apache.org/repos/asf/ignite/blob/4d9f4504/modules/web-console/frontend/app/components/ignite-chart-series-selector/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ignite-chart-series-selector/controller.js b/modules/web-console/frontend/app/components/ignite-chart-series-selector/controller.js
new file mode 100644
index 0000000..00f57ca
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-chart-series-selector/controller.js
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default class IgniteChartSeriesSelectorController {
+ static $inject = [];
+
+ constructor() {
+ this.charts = [];
+ this.selectedCharts = [];
+ }
+
+ $onChanges(changes) {
+ if (changes && 'chartApi' in changes && changes.chartApi.currentValue)
+ this.applyValues();
+ }
+
+ applyValues() {
+ this.charts = this._makeMenu();
+ this.selectedCharts = this.charts.map(({ key }) => key);
+ }
+
+ setSelectedCharts() {
+ const selectedDataset = ({ label }) => this.selectedCharts.includes(label);
+
+ this.chartApi.config.data.datasets
+ .forEach((dataset) => {
+ dataset.hidden = true;
+
+ if (!selectedDataset(dataset))
+ return;
+
+ dataset.hidden = false;
+ });
+
+ this.chartApi.update();
+ }
+
+ _makeMenu() {
+ const labels = this.chartApi.config.datasetLegendMapping;
+
+ return Object.keys(this.chartApi.config.datasetLegendMapping).map((key) => {
+ return {
+ key,
+ label: labels[key]
+ };
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4d9f4504/modules/web-console/frontend/app/components/ignite-chart-series-selector/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ignite-chart-series-selector/index.js b/modules/web-console/frontend/app/components/ignite-chart-series-selector/index.js
new file mode 100644
index 0000000..7ad3da0
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-chart-series-selector/index.js
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import component from './component';
+
+export default angular
+ .module('ignite-console.ignite-chart-series-selector', [])
+ .component('igniteChartSeriesSelector', component);
http://git-wip-us.apache.org/repos/asf/ignite/blob/4d9f4504/modules/web-console/frontend/app/components/ignite-chart-series-selector/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ignite-chart-series-selector/template.pug b/modules/web-console/frontend/app/components/ignite-chart-series-selector/template.pug
new file mode 100644
index 0000000..fec0d9a
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-chart-series-selector/template.pug
@@ -0,0 +1,29 @@
+//-
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+button.btn-ignite.btn-ignite--link-dashed-secondary(
+ protect-from-bs-select-render
+ bs-select
+ ng-model='$ctrl.selectedCharts'
+ ng-change='$ctrl.setSelectedCharts()'
+ ng-model-options='{debounce: {default: 5}}',
+ bs-options='chart.key as chart.label for chart in $ctrl.charts'
+ bs-on-before-show='$ctrl.onShow'
+ data-multiple='true'
+ ng-transclude
+ ng-show='$ctrl.charts.length'
+)
+ svg(ignite-icon='gear').icon
http://git-wip-us.apache.org/repos/asf/ignite/blob/4d9f4504/modules/web-console/frontend/app/components/ignite-chart/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ignite-chart/controller.js b/modules/web-console/frontend/app/components/ignite-chart/controller.js
new file mode 100644
index 0000000..df11050
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-chart/controller.js
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import _ from 'lodash';
+
+/**
+ * @typedef {{x: number, y: {[key: string]: number}}} IgniteChartDataPoint
+ */
+
+const RANGE_RATE_PRESET = [{
+ label: '1 min',
+ value: 1
+}, {
+ label: '5 min',
+ value: 5
+}, {
+ label: '10 min',
+ value: 10
+}, {
+ label: '15 min',
+ value: 15
+}, {
+ label: '30 min',
+ value: 30
+}];
+
+export class IgniteChartController {
+ /** @type {import('chart.js').ChartConfiguration} */
+ chartOptions;
+ /** @type {string} */
+ chartTitle;
+ /** @type {IgniteChartDataPoint} */
+ chartDataPoint;
+ /** @type {Array<IgniteChartDataPoint>} */
+ chartHistory;
+ newPoints = [];
+
+ static $inject = ['$element', 'IgniteChartColors', '$filter'];
+
+ /**
+ * @param {JQLite} $element
+ * @param {ng.IScope} $scope
+ * @param {ng.IFilterService} $filter
+ */
+ constructor($element, IgniteChartColors, $filter) {
+ this.$element = $element;
+ this.IgniteChartColors = IgniteChartColors;
+
+ this.datePipe = $filter('date');
+ this.ranges = RANGE_RATE_PRESET;
+ this.currentRange = this.ranges[0];
+ this.maxRangeInMilliseconds = RANGE_RATE_PRESET[RANGE_RATE_PRESET.length - 1].value * 60 * 1000;
+ this.ctx = this.$element.find('canvas')[0].getContext('2d');
+
+ this.localHistory = [];
+ this.updateIsBusy = false;
+ }
+
+ $onDestroy() {
+ if (this.chart) this.chart.destroy();
+ this.$element = this.ctx = this.chart = null;
+ }
+
+ $onInit() {
+ this.chartColors = _.get(this.chartOptions, 'chartColors', this.IgniteChartColors);
+ }
+
+ /**
+ * @param {{chartOptions: ng.IChangesObject<import('chart.js').ChartConfiguration>, chartTitle: ng.IChangesObject<string>, chartDataPoint: ng.IChangesObject<IgniteChartDataPoint>, chartHistory: ng.IChangesObject<Array<IgniteChartDataPoint>>}} changes
+ */
+ $onChanges(changes) {
+ if (this.chart && _.get(changes, 'refreshRate.currentValue'))
+ this.onRefreshRateChanged(_.get(changes, 'refreshRate.currentValue'));
+
+ // TODO: Investigate other signaling for resetting component state.
+ if (changes.chartDataPoint && _.isNil(changes.chartDataPoint.currentValue)) {
+ this.clearDatasets();
+ return;
+ }
+
+ if (changes.chartHistory && changes.chartHistory.currentValue && changes.chartHistory.currentValue.length !== changes.chartHistory.previousValue.length) {
+ if (!this.chart)
+ this.initChart();
+
+ this.clearDatasets();
+ this.localHistory = [...changes.chartHistory.currentValue];
+
+ this.newPoints.splice(0, this.newPoints.length, ...changes.chartHistory.currentValue);
+
+ this.onRefresh();
+ this.rerenderChart();
+ return;
+ }
+
+ if (this.chartDataPoint && changes.chartDataPoint) {
+ if (!this.chart)
+ this.initChart();
+
+ this.newPoints.push(this.chartDataPoint);
+ this.localHistory.push(this.chartDataPoint);
+ }
+ }
+
+ async initChart() {
+ /** @type {import('chart.js').ChartConfiguration} */
+ this.config = {
+ type: 'line',
+ data: {
+ datasets: []
+ },
+ options: {
+ elements: {
+ line: {
+ tension: 0
+ },
+ point: {
+ radius: 2,
+ pointStyle: 'rectRounded'
+ }
+ },
+ animation: {
+ duration: 0 // general animation time
+ },
+ hover: {
+ animationDuration: 0 // duration of animations when hovering an item
+ },
+ responsiveAnimationDuration: 0, // animation duration after a resize
+ maintainAspectRatio: false,
+ responsive: true,
+ legend: {
+ display: false
+ },
+ scales: {
+ xAxes: [{
+ type: 'realtime',
+ display: true,
+ time: {
+ displayFormats: {
+ second: 'HH:mm:ss',
+ minute: 'HH:mm:ss',
+ hour: 'HH:mm:ss'
+ }
+ },
+ ticks: {
+ maxRotation: 0,
+ minRotation: 0
+ }
+ }],
+ yAxes: [{
+ type: 'linear',
+ display: true,
+ ticks: {
+ min: 0,
+ beginAtZero: true,
+ maxTicksLimit: 4,
+ callback: (value, index, labels) => {
+ if (value === 0)
+ return 0;
+
+ if (_.max(labels) <= 4000 && value <= 4000)
+ return value;
+
+ if (_.max(labels) <= 1000000 && value <= 1000000)
+ return `${value / 1000}K`;
+
+ if ((_.max(labels) <= 4000000 && value >= 500000) || (_.max(labels) > 4000000))
+ return `${value / 1000000}M`;
+
+ return value;
+ }
+ }
+ }]
+ },
+ tooltips: {
+ mode: 'index',
+ position: 'nearest',
+ intersect: false,
+ xPadding: 20,
+ yPadding: 20,
+ bodyFontSize: 13,
+ callbacks: {
+ title: (tooltipItem) => {
+ return tooltipItem[0].xLabel.slice(0, -7);
+ },
+ label: (tooltipItem, data) => {
+ const label = data.datasets[tooltipItem.datasetIndex].label || '';
+
+ return `${_.startCase(label)}: ${tooltipItem.yLabel} per sec`;
+ },
+ labelColor: (tooltipItem) => {
+ return {
+ borderColor: 'rgba(255,255,255,0.5)',
+ borderWidth: 0,
+ boxShadow: 'none',
+ backgroundColor: this.chartColors[tooltipItem.datasetIndex]
+ };
+ }
+ }
+ },
+ plugins: {
+ streaming: {
+ duration: this.currentRange.value * 1000 * 60,
+ frameRate: 1000 / this.refreshRate || 1 / 3,
+ refresh: this.refreshRate || 3000,
+ onRefresh: () => {
+ this.onRefresh();
+ }
+ }
+ }
+ }
+ };
+
+ this.config = _.merge(this.config, this.chartOptions);
+
+ const chartModule = await import('chart.js');
+ const Chart = chartModule.default;
+
+ await import('chartjs-plugin-streaming');
+
+ this.chart = new Chart(this.ctx, this.config);
+ this.changeXRange(this.currentRange);
+ }
+
+ onRefresh() {
+ this.newPoints.forEach((point) => {
+ this.appendChartPoint(point);
+ });
+
+ this.newPoints.splice(0, this.newPoints.length);
+ }
+
+ /**
+ * @param {IgniteChartDataPoint} dataPoint
+ */
+ appendChartPoint(dataPoint) {
+ Object.keys(dataPoint.y).forEach((key) => {
+ if (this.checkDatasetCanBeAdded(key)) {
+ let datasetIndex = this.findDatasetIndex(key);
+
+ if (datasetIndex < 0) {
+ datasetIndex = this.config.data.datasets.length;
+ this.addDataset(key);
+ }
+
+ // Prune excessive data points.
+ if (this.maxPointsNumber && this.config.data.datasets[datasetIndex].length - this.maxPointsNumber > 0)
+ this.config.data.datasets[datasetIndex].data.splice(0, this.config.data.datasets[datasetIndex].length - this.maxPointsNumber);
+
+ this.config.data.datasets[datasetIndex].data.push({x: dataPoint.x, y: dataPoint.y[key]});
+ this.config.data.datasets[datasetIndex].borderColor = this.chartOptions.chartColors[datasetIndex];
+ this.config.data.datasets[datasetIndex].borderWidth = 2;
+ this.config.data.datasets[datasetIndex].fill = false;
+ }
+ });
+ }
+
+ /**
+ * Checks if a key of dataset can be added to chart or should be ignored.
+ * @param dataPointKey {String}
+ * @return {Boolean}
+ */
+ checkDatasetCanBeAdded(dataPointKey) {
+ // If datasetLegendMapping is empty all keys are allowed.
+ if (!this.config.datasetLegendMapping)
+ return true;
+
+ return Object.keys(this.config.datasetLegendMapping).includes(dataPointKey);
+ }
+
+ clearDatasets() {
+ if (!_.isNil(this.config))
+ this.config.data.datasets.forEach((dataset) => dataset.data = []);
+ }
+
+ addDataset(datasetName) {
+ if (this.findDatasetIndex(datasetName) >= 0)
+ throw new Error(`Dataset with name ${datasetName} is already in chart`);
+ else
+ this.config.data.datasets.push({ label: datasetName, data: [] });
+ }
+
+ findDatasetIndex(searchedDatasetLabel) {
+ return this.config.data.datasets.findIndex((dataset) => dataset.label === searchedDatasetLabel);
+ }
+
+ changeXRange(range) {
+ const deltaInMilliSeconds = range.value * 60 * 1000;
+ this.chart.config.options.plugins.streaming.duration = deltaInMilliSeconds;
+
+ this.clearDatasets();
+ this.newPoints.splice(0, this.newPoints.length, ...this.localHistory);
+
+ this.onRefresh();
+ this.rerenderChart();
+ }
+
+ onRefreshRateChanged(refreshRate) {
+ this.chart.config.options.plugins.streaming.frameRate = 1000 / refreshRate;
+ this.chart.config.options.plugins.streaming.refresh = refreshRate;
+ this.rerenderChart();
+ }
+
+ rerenderChart() {
+ this.chart.update();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4d9f4504/modules/web-console/frontend/app/components/ignite-chart/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ignite-chart/index.js b/modules/web-console/frontend/app/components/ignite-chart/index.js
new file mode 100644
index 0000000..337ba36
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-chart/index.js
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import { IgniteChartController } from './controller';
+import template from './template.pug';
+import './style.scss';
+
+export default angular
+ .module('ignite-console.ignite-chart', [])
+ .component('igniteChart', {
+ controller: IgniteChartController,
+ template,
+ bindings: {
+ chartOptions: '<',
+ chartDataPoint: '<',
+ chartHistory: '<',
+ chartTitle: '<',
+ chartColors: '<',
+ maxPointsNumber: '<',
+ refreshRate: '<'
+ }
+ });
http://git-wip-us.apache.org/repos/asf/ignite/blob/4d9f4504/modules/web-console/frontend/app/components/ignite-chart/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ignite-chart/style.scss b/modules/web-console/frontend/app/components/ignite-chart/style.scss
new file mode 100644
index 0000000..be0fb6d
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-chart/style.scss
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ignite-chart {
+ display: flex;
+ flex-direction: column;
+
+ height: 100%;
+
+ header {
+ display: flex;
+ flex-direction: row;
+ justify-content: space-between;
+ box-sizing: content-box;
+
+ height: 36px;
+ min-height: 36px;
+ padding: 7px 21px;
+
+ border-bottom: 1px solid #d4d4d4;
+
+ h5 {
+ margin: 0;
+
+ font-size: 16px;
+ font-weight: normal;
+ line-height: 36px;
+ }
+
+ > div {
+ display: flex;
+ align-items: center;
+ }
+ }
+
+ .ignite-chart-placeholder {
+ display: block;
+ height: calc(100% - 71px);
+ margin-top: 20px;
+ }
+
+ .no-data {
+ position: absolute;
+ top: 50%;
+
+ width: 100%;
+ padding: 0 20px;
+
+ border-radius: 0 0 4px 4px;
+
+ font-style: italic;
+ line-height: 16px;
+ text-align: center;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4d9f4504/modules/web-console/frontend/app/components/ignite-chart/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ignite-chart/template.pug b/modules/web-console/frontend/app/components/ignite-chart/template.pug
new file mode 100644
index 0000000..076d8d4
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-chart/template.pug
@@ -0,0 +1,36 @@
+//-
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+header
+ div
+ h5 {{ $ctrl.chartTitle }}
+ ignite-chart-series-selector(chart-api='$ctrl.chart')
+
+ div
+ span Range:
+ button.btn-ignite.btn-ignite--link-success.link-primary(
+ ng-model='$ctrl.currentRange'
+ ng-change='$ctrl.changeXRange($ctrl.currentRange)'
+ bs-options='item as item.label for item in $ctrl.ranges'
+ bs-select
+ )
+
+.ignite-chart-placeholder
+ canvas
+
+.no-data(ng-if='!$ctrl.config.data.datasets')
+ | No Data #[br]
+ | Make sure you are connected to the right grid.
http://git-wip-us.apache.org/repos/asf/ignite/blob/4d9f4504/modules/web-console/frontend/app/primitives/btn/index.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/primitives/btn/index.scss b/modules/web-console/frontend/app/primitives/btn/index.scss
index 2d9e9c4..277b1ae 100644
--- a/modules/web-console/frontend/app/primitives/btn/index.scss
+++ b/modules/web-console/frontend/app/primitives/btn/index.scss
@@ -319,10 +319,19 @@ $btn-content-padding-with-border: 9px 11px;
$line-color: $ignite-brand-success;
border-right-color: change-color($line-color, $saturation: 63%, $lightness: 33%);
}
+
+ &[disabled] .btn-ignite.btn-ignite--primary {
+ border-right-color: change-color($ignite-brand-primary, $lightness: 83%);
+ }
+
+ &[disabled] .btn-ignite.btn-ignite--success {
+ border-right-color: change-color($ignite-brand-success, $lightness: 83%);
+ }
}
@mixin ignite-link($color, $color-hover) {
color: $color;
+ text-decoration: none;
&:hover, &.hover,
&:focus, &.focus {
http://git-wip-us.apache.org/repos/asf/ignite/blob/4d9f4504/modules/web-console/frontend/package.json
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 02acbe0..6a6b97f 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -54,6 +54,8 @@
"brace": "0.10.0",
"browser-update": "3.1.13",
"bson-objectid": "1.1.5",
+ "chart.js": "2.7.2",
+ "chartjs-plugin-streaming": "1.5.0",
"file-saver": "1.3.3",
"font-awesome": "4.7.0",
"jquery": "3.2.1",
[05/50] [abbrv] ignite git commit: IGNITE-9104 Web Console: Enhanced
UI grid.
Posted by av...@apache.org.
IGNITE-9104 Web Console: Enhanced UI grid.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1b5d2755
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1b5d2755
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1b5d2755
Branch: refs/heads/ignite-8446
Commit: 1b5d27555383a7c70801af01d7d663c31adc715a
Parents: 7c4705c
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Tue Jul 31 10:19:33 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Jul 31 10:19:33 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/app/app.js | 2 +
.../app/components/grid-export/index.js | 2 +-
.../app/components/grid-export/style.scss | 34 ++++++++++++++
.../app/components/grid-export/template.pug | 6 ++-
.../components/grid-item-selected/template.pug | 5 ++-
.../app/components/grid-no-data/controller.js | 7 +--
.../components/grid-showing-rows/component.js | 29 ++++++++++++
.../components/grid-showing-rows/controller.js | 47 ++++++++++++++++++++
.../app/components/grid-showing-rows/index.js | 24 ++++++++++
.../app/components/grid-showing-rows/style.scss | 20 +++++++++
10 files changed, 168 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b5d2755/modules/web-console/frontend/app/app.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index b9d72ab..baf06dd 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -126,6 +126,7 @@ import gridColumnSelector from './components/grid-column-selector';
import gridItemSelected from './components/grid-item-selected';
import gridNoData from './components/grid-no-data';
import gridExport from './components/grid-export';
+import gridShowingRows from './components/grid-showing-rows';
import bsSelectMenu from './components/bs-select-menu';
import protectFromBsSelectRender from './components/protect-from-bs-select-render';
import uiGridHovering from './components/ui-grid-hovering';
@@ -221,6 +222,7 @@ export default angular.module('ignite-console', [
gridItemSelected.name,
gridNoData.name,
gridExport.name,
+ gridShowingRows.name,
bsSelectMenu.name,
uiGridHovering.name,
uiGridFilters.name,
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b5d2755/modules/web-console/frontend/app/components/grid-export/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-export/index.js b/modules/web-console/frontend/app/components/grid-export/index.js
index 9fa8835..00e2bce 100644
--- a/modules/web-console/frontend/app/components/grid-export/index.js
+++ b/modules/web-console/frontend/app/components/grid-export/index.js
@@ -16,7 +16,7 @@
*/
import angular from 'angular';
-
+import './style.scss';
import component from './component';
export default angular
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b5d2755/modules/web-console/frontend/app/components/grid-export/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-export/style.scss b/modules/web-console/frontend/app/components/grid-export/style.scss
new file mode 100644
index 0000000..29e35fd
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-export/style.scss
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+grid-export {
+ &:not(.link-success) {
+ button:nth-child(2) {
+ display: none;
+ }
+ }
+
+ &.link-success {
+ button:nth-child(1) {
+ display: none;
+ }
+ }
+
+ [ignite-icon='download'] {
+ margin-right: 5px;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b5d2755/modules/web-console/frontend/app/components/grid-export/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-export/template.pug b/modules/web-console/frontend/app/components/grid-export/template.pug
index fac10df..a085db6 100644
--- a/modules/web-console/frontend/app/components/grid-export/template.pug
+++ b/modules/web-console/frontend/app/components/grid-export/template.pug
@@ -14,5 +14,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-button.btn-ignite.btn-ignite--link-dashed-secondary(ng-click='$ctrl.export()' bs-tooltip='' data-title='Export filtered rows to CSV' data-placement='top')
+button.btn-ignite.btn-ignite--primary-outline(ng-click='$ctrl.export()' bs-tooltip='' data-title='Export filtered rows to CSV' data-placement='top')
+ svg(ignite-icon='csv')
+
+button.btn-ignite.btn-ignite--link-success.link-success(ng-click='$ctrl.export()' bs-tooltip='' data-title='Export filtered rows to CSV' data-placement='top')
svg(ignite-icon='download')
+ i Export to .csv
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b5d2755/modules/web-console/frontend/app/components/grid-item-selected/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-item-selected/template.pug b/modules/web-console/frontend/app/components/grid-item-selected/template.pug
index eca079e..cf0695b 100644
--- a/modules/web-console/frontend/app/components/grid-item-selected/template.pug
+++ b/modules/web-console/frontend/app/components/grid-item-selected/template.pug
@@ -14,4 +14,7 @@
See the License for the specific language governing permissions and
limitations under the License.
-i {{ $ctrl.selected }} of {{ $ctrl.count }} selected
+span(ng-if='$ctrl.count')
+ i {{ $ctrl.selected }} of {{ $ctrl.count }} selected
+span(ng-if='!$ctrl.count')
+ i Showing: 0 rows
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b5d2755/modules/web-console/frontend/app/components/grid-no-data/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-no-data/controller.js b/modules/web-console/frontend/app/components/grid-no-data/controller.js
index 2451e03..d252c58 100644
--- a/modules/web-console/frontend/app/components/grid-no-data/controller.js
+++ b/modules/web-console/frontend/app/components/grid-no-data/controller.js
@@ -15,9 +15,7 @@
* limitations under the License.
*/
-import filter from 'lodash/fp/filter';
-
-const rowsFiltered = filter(({ visible }) => visible);
+import _ from 'lodash';
export default class {
static $inject = ['$scope', 'uiGridConstants'];
@@ -47,7 +45,6 @@ export default class {
this.noData = false;
- const filtered = rowsFiltered(this.gridApi.grid.rows);
- this.noDataFiltered = !filtered.length;
+ this.noDataFiltered = _.sumBy(this.gridApi.grid.rows, 'visible') === 0;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b5d2755/modules/web-console/frontend/app/components/grid-showing-rows/component.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-showing-rows/component.js b/modules/web-console/frontend/app/components/grid-showing-rows/component.js
new file mode 100644
index 0000000..93fa1b1
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-showing-rows/component.js
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './style.scss';
+import controller from './controller.js';
+
+export default {
+ template: `
+ <i>Showing: {{ $ctrl.count }} rows</i>
+ `,
+ controller,
+ bindings: {
+ gridApi: '<'
+ }
+};
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b5d2755/modules/web-console/frontend/app/components/grid-showing-rows/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-showing-rows/controller.js b/modules/web-console/frontend/app/components/grid-showing-rows/controller.js
new file mode 100644
index 0000000..cb5b496
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-showing-rows/controller.js
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import _ from 'lodash';
+
+export default class {
+ static $inject = ['$scope', 'uiGridConstants'];
+
+ constructor($scope, uiGridConstants) {
+ Object.assign(this, {$scope, uiGridConstants});
+
+ this.count = 0;
+ }
+
+ $onChanges(changes) {
+ if (changes && 'gridApi' in changes && changes.gridApi.currentValue) {
+ this.applyValues();
+
+ this.gridApi.core.on.rowsVisibleChanged(this.$scope, () => {
+ this.applyValues();
+ });
+ }
+ }
+
+ applyValues() {
+ if (!this.gridApi.grid.rows.length) {
+ this.count = 0;
+ return;
+ }
+
+ this.count = _.sumBy(this.gridApi.grid.rows, 'visible');
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b5d2755/modules/web-console/frontend/app/components/grid-showing-rows/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-showing-rows/index.js b/modules/web-console/frontend/app/components/grid-showing-rows/index.js
new file mode 100644
index 0000000..9f8c921
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-showing-rows/index.js
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import component from './component';
+
+export default angular
+ .module('ignite-console.grid-showing-rows', [])
+ .component('gridShowingRows', component);
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b5d2755/modules/web-console/frontend/app/components/grid-showing-rows/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-showing-rows/style.scss b/modules/web-console/frontend/app/components/grid-showing-rows/style.scss
new file mode 100644
index 0000000..a37f1b5
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-showing-rows/style.scss
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+grid-showing-rows {
+ color: #757575;
+}
[36/50] [abbrv] ignite git commit: ignite-6467 Result of
updateRebalanceVersion was incorrect when diffFromAffinityVer did not match
current affinity version.
Posted by av...@apache.org.
ignite-6467 Result of updateRebalanceVersion was incorrect when diffFromAffinityVer did not match current affinity version.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5fc18e6f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5fc18e6f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5fc18e6f
Branch: refs/heads/ignite-8446
Commit: 5fc18e6f1f81643747bcc8d5ef8ee4fe47116763
Parents: 891da2a
Author: sboikov <sb...@apache.org>
Authored: Wed Aug 1 23:51:19 2018 +0300
Committer: sboikov <sb...@apache.org>
Committed: Wed Aug 1 23:51:19 2018 +0300
----------------------------------------------------------------------
.../dht/GridDhtPartitionTopologyImpl.java | 23 +++++++++++++-------
1 file changed, 15 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5fc18e6f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index aa43b2f..c5c574f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -433,7 +433,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
}
- updateRebalanceVersion(affAssignment);
+ updateRebalanceVersion(affVer, affAssignment);
return needRefresh;
}
@@ -762,7 +762,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (node2part != null && node2part.valid())
changed |= checkEvictions(updateSeq, aff);
- updateRebalanceVersion(aff.assignment());
+ updateRebalanceVersion(aff.topologyVersion(), aff.assignment());
consistencyCheck();
@@ -1535,7 +1535,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (exchangeVer == null)
changed |= checkEvictions(updateSeq, aff);
- updateRebalanceVersion(aff.assignment());
+ updateRebalanceVersion(aff.topologyVersion(), aff.assignment());
}
if (partSizes != null)
@@ -1803,7 +1803,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (exchId == null)
changed |= checkEvictions(updateSeq, aff);
- updateRebalanceVersion(aff.assignment());
+ updateRebalanceVersion(aff.topologyVersion(), aff.assignment());
}
consistencyCheck();
@@ -1827,8 +1827,8 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
/** {@inheritDoc} */
@Override public void onExchangeDone(@Nullable GridDhtPartitionsExchangeFuture fut,
- AffinityAssignment assignment,
- boolean updateRebalanceVer) {
+ AffinityAssignment assignment,
+ boolean updateRebalanceVer) {
lock.writeLock().lock();
try {
@@ -1850,10 +1850,13 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
else
diffFromAffinityVer = readyTopVer;
+
+ if (!updateRebalanceVer)
+ updateRebalanceVersion(assignment.topologyVersion(), assignment.assignment());
}
if (updateRebalanceVer)
- updateRebalanceVersion(assignment.assignment());
+ updateRebalanceVersion(assignment.topologyVersion(), assignment.assignment());
}
finally {
lock.writeLock().unlock();
@@ -2748,9 +2751,13 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
/**
+ * @param affVer Affinity version.
* @param aff Affinity assignments.
*/
- private void updateRebalanceVersion(List<List<ClusterNode>> aff) {
+ private void updateRebalanceVersion(AffinityTopologyVersion affVer, List<List<ClusterNode>> aff) {
+ if (!grp.isReplicated() && !affVer.equals(diffFromAffinityVer))
+ return;
+
if (!rebalancedTopVer.equals(readyTopVer)) {
if (node2part == null || !node2part.valid())
return;
[39/50] [abbrv] ignite git commit: IGNITE-8715 Problems with
Closeable objects from Factory
Posted by av...@apache.org.
IGNITE-8715 Problems with Closeable objects from Factory
Signed-off-by: Anton Vinogradov <av...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d632eb6f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d632eb6f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d632eb6f
Branch: refs/heads/ignite-8446
Commit: d632eb6f8d5f0f23111042dfab3274ed20052a4a
Parents: 1cdf7ef
Author: pereslegin-pa <xx...@gmail.com>
Authored: Thu Aug 2 14:19:47 2018 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Thu Aug 2 14:19:47 2018 +0300
----------------------------------------------------------------------
.../internal/processors/cache/GridCacheContext.java | 4 ++++
.../cache/GridCacheLoaderWriterStore.java | 6 ++++++
.../continuous/CacheContinuousQueryManager.java | 16 ++++++++++++++--
3 files changed, 24 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d632eb6f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index 191734b..61b1878 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache;
+import java.io.Closeable;
import java.io.Externalizable;
import java.io.IOException;
import java.io.InvalidObjectException;
@@ -2011,6 +2012,9 @@ public class GridCacheContext<K, V> implements Externalizable {
dataStructuresMgr = null;
cacheObjCtx = null;
+ if (expiryPlc instanceof Closeable)
+ U.closeQuiet((Closeable)expiryPlc);
+
mgrs.clear();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d632eb6f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java
index 03beaf0..cf07b18 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache;
+import java.io.Closeable;
import java.io.Serializable;
import java.util.Collection;
import java.util.Collections;
@@ -26,6 +27,7 @@ import javax.cache.integration.CacheLoader;
import javax.cache.integration.CacheWriter;
import org.apache.ignite.cache.store.CacheStore;
import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiInClosure;
import org.apache.ignite.lifecycle.LifecycleAware;
import org.jetbrains.annotations.Nullable;
@@ -81,9 +83,13 @@ class GridCacheLoaderWriterStore<K, V> implements CacheStore<K, V>, LifecycleAwa
@Override public void stop() {
if (ldr instanceof LifecycleAware)
((LifecycleAware)ldr).stop();
+ else if (ldr instanceof Closeable)
+ U.closeQuiet((Closeable)ldr);
if (writer instanceof LifecycleAware)
((LifecycleAware)writer).stop();
+ else if (writer instanceof Closeable)
+ U.closeQuiet((Closeable)writer);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/d632eb6f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 1cdc014..c7635e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache.query.continuous;
+import java.io.Closeable;
import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
@@ -954,6 +955,9 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
/** */
private volatile UUID routineId;
+ /** */
+ private volatile JCacheQueryLocalListener locLsnr;
+
/**
* @param cfg Listener configuration.
* @param onStart {@code True} if executed on cache start.
@@ -989,7 +993,7 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
final byte types0 = types;
- final CacheEntryUpdatedListener locLsnr = new JCacheQueryLocalListener(
+ locLsnr = new JCacheQueryLocalListener(
locLsnrImpl,
log);
@@ -1063,13 +1067,15 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
cctx.kernalContext().continuous().stopRoutine(routineId0).get();
cctx.config().removeCacheEntryListenerConfiguration(cfg);
+
+ U.closeQuiet(locLsnr);
}
}
/**
*
*/
- static class JCacheQueryLocalListener<K, V> implements CacheEntryUpdatedListener<K, V> {
+ static class JCacheQueryLocalListener<K, V> implements CacheEntryUpdatedListener<K, V>, Closeable {
/** */
final CacheEntryListener<K, V> impl;
@@ -1154,6 +1160,12 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
protected boolean async() {
return U.hasAnnotation(impl, IgniteAsyncCallback.class);
}
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IOException {
+ if (impl instanceof Closeable)
+ U.closeQuiet((Closeable)impl);
+ }
}
/**
[34/50] [abbrv] ignite git commit: IGNITE-7165 Re-balancing is
cancelled if client node joins
Posted by av...@apache.org.
IGNITE-7165 Re-balancing is cancelled if client node joins
Signed-off-by: Anton Vinogradov <av...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/137dd06a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/137dd06a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/137dd06a
Branch: refs/heads/ignite-8446
Commit: 137dd06aaee9cc84104e6b4bf48306b050341e3a
Parents: f6f731f
Author: Maxim Muzafarov <ma...@gmail.com>
Authored: Wed Aug 1 18:39:54 2018 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Wed Aug 1 18:39:54 2018 +0300
----------------------------------------------------------------------
.../GridCachePartitionExchangeManager.java | 68 ++++++---
.../processors/cache/GridCachePreloader.java | 21 ++-
.../cache/GridCachePreloaderAdapter.java | 6 +
.../dht/preloader/GridDhtPartitionDemander.java | 55 ++++---
.../dht/preloader/GridDhtPartitionSupplier.java | 26 ++--
.../dht/preloader/GridDhtPreloader.java | 60 +++++++-
.../preloader/GridDhtPreloaderAssignments.java | 6 +-
.../ClusterBaselineNodesMetricsSelfTest.java | 1 -
.../cache/CacheValidatorMetricsTest.java | 4 +-
.../dht/GridCacheDhtPreloadSelfTest.java | 68 +--------
.../atomic/IgniteCacheAtomicProtocolTest.java | 3 -
.../GridCacheRebalancingAsyncSelfTest.java | 7 +-
.../GridCacheRebalancingCancelTest.java | 106 +++++++++++++
...idCacheRebalancingPartitionCountersTest.java | 3 +-
.../GridCacheRebalancingSyncSelfTest.java | 149 +++++++------------
...lientAffinityAssignmentWithBaselineTest.java | 4 +-
...SlowHistoricalRebalanceSmallHistoryTest.java | 5 +-
...lFlushMultiNodeFailoverAbstractSelfTest.java | 2 +-
.../GridMarshallerMappingConsistencyTest.java | 3 +-
.../junits/common/GridCommonAbstractTest.java | 115 +++-----------
.../testsuites/IgniteCacheTestSuite3.java | 2 +
21 files changed, 370 insertions(+), 344 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 05eeee3..824aa67 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -66,7 +66,6 @@ import org.apache.ignite.internal.managers.discovery.DiscoveryLocalJoinData;
import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
@@ -88,6 +87,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.Ign
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionsToReloadMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.RebalanceReassignExchangeTask;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.StopCachesOnClientReconnectExchangeTask;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
@@ -179,6 +179,14 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
private final ConcurrentMap<AffinityTopologyVersion, AffinityReadyFuture> readyFuts =
new ConcurrentSkipListMap<>();
+ /**
+ * Latest started rebalance topology version but possibly not finished yet. Value {@code NONE}
+ * means that previous rebalance is undefined and the new one should be initiated.
+ *
+ * Should not be used to determine latest rebalanced topology.
+ */
+ private volatile AffinityTopologyVersion rebTopVer = AffinityTopologyVersion.NONE;
+
/** */
private GridFutureAdapter<?> reconnectExchangeFut;
@@ -827,6 +835,13 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
}
/**
+ * @return Latest rebalance topology version or {@code NONE} if there is no info.
+ */
+ public AffinityTopologyVersion rebalanceTopologyVersion() {
+ return rebTopVer;
+ }
+
+ /**
* @return Last initialized topology future.
*/
public GridDhtPartitionsExchangeFuture lastTopologyFuture() {
@@ -2558,6 +2573,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
if (grp.isLocal())
continue;
+ if (grp.preloader().rebalanceRequired(rebTopVer, exchFut))
+ rebTopVer = AffinityTopologyVersion.NONE;
+
changed |= grp.topology().afterExchange(exchFut);
}
@@ -2565,7 +2583,11 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
refreshPartitions();
}
- if (!cctx.kernalContext().clientNode()) {
+ // Schedule rebalance if force rebalance or force reassign occurs.
+ if (exchFut == null)
+ rebTopVer = AffinityTopologyVersion.NONE;
+
+ if (!cctx.kernalContext().clientNode() && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
assignsMap = new HashMap<>();
IgniteCacheSnapshotManager snp = cctx.snapshot();
@@ -2582,6 +2604,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
assigns = grp.preloader().generateAssignments(exchId, exchFut);
assignsMap.put(grp.groupId(), assigns);
+
+ if (resVer == null)
+ resVer = grp.topology().readyTopologyVersion();
}
}
}
@@ -2590,7 +2615,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
busy = false;
}
- if (assignsMap != null) {
+ if (assignsMap != null && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
int size = assignsMap.size();
NavigableMap<Integer, List<Integer>> orderMap = new TreeMap<>();
@@ -2628,11 +2653,6 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
if (assigns != null)
assignsCancelled |= assigns.cancelled();
- // Cancels previous rebalance future (in case it's not done yet).
- // Sends previous rebalance stopped event (if necessary).
- // Creates new rebalance future.
- // Sends current rebalance started event (if necessary).
- // Finishes cache sync future (on empty assignments).
Runnable cur = grp.preloader().addAssignments(assigns,
forcePreload,
cnt,
@@ -2650,7 +2670,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
if (forcedRebFut != null)
forcedRebFut.markInitialized();
- if (assignsCancelled) { // Pending exchange.
+ if (assignsCancelled || hasPendingExchange()) {
U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
"[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
", node=" + exchId.nodeId() + ']');
@@ -2658,25 +2678,31 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
else if (r != null) {
Collections.reverse(rebList);
- U.log(log, "Rebalancing scheduled [order=" + rebList + "]");
+ U.log(log, "Rebalancing scheduled [order=" + rebList +
+ ", top=" + resVer + ", force=" + (exchFut == null) +
+ ", evt=" + exchId.discoveryEventName() +
+ ", node=" + exchId.nodeId() + ']');
- if (!hasPendingExchange()) {
- U.log(log, "Rebalancing started " +
- "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
- ", node=" + exchId.nodeId() + ']');
+ rebTopVer = resVer;
- r.run(); // Starts rebalancing routine.
- }
- else
- U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
- "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
- ", node=" + exchId.nodeId() + ']');
+ // Start rebalancing cache groups chain. Each group will be rebalanced
+ // sequentially one by one e.g.:
+ // ignite-sys-cache -> cacheGroupR1 -> cacheGroupP2 -> cacheGroupR3
+ r.run();
}
else
U.log(log, "Skipping rebalancing (nothing scheduled) " +
- "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
+ "[top=" + resVer + ", force=" + (exchFut == null) +
+ ", evt=" + exchId.discoveryEventName() +
", node=" + exchId.nodeId() + ']');
}
+ else
+ U.log(log, "Skipping rebalancing (no affinity changes) " +
+ "[top=" + resVer +
+ ", rebTopVer=" + rebTopVer +
+ ", evt=" + exchId.discoveryEventName() +
+ ", evtNode=" + exchId.nodeId() +
+ ", client=" + cctx.kernalContext().clientNode() + ']');
}
catch (IgniteInterruptedCheckedException e) {
throw e;
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index 5fa7a82..d629e94 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -24,6 +24,7 @@ import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.ForceRebalanceExchangeTask;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
@@ -63,9 +64,16 @@ public interface GridCachePreloader {
public void onInitialExchangeComplete(@Nullable Throwable err);
/**
+ * @param rebTopVer Previous rebalance topology version or {@code NONE} if there is no info.
+ * @param exchFut Completed exchange future.
+ * @return {@code True} if rebalance should be started (previous will be interrupted).
+ */
+ public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer, GridDhtPartitionsExchangeFuture exchFut);
+
+ /**
* @param exchId Exchange ID.
- * @param exchFut Exchange future.
- * @return Assignments or {@code null} if detected that there are pending exchanges.
+ * @param exchFut Completed exchange future. Can be {@code null} if forced or reassigned generation occurs.
+ * @return Partition assignments which will be requested from supplier nodes.
*/
@Nullable public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
@Nullable GridDhtPartitionsExchangeFuture exchFut);
@@ -74,10 +82,10 @@ public interface GridCachePreloader {
* Adds assignments to preloader.
*
* @param assignments Assignments to add.
- * @param forcePreload Force preload flag.
- * @param rebalanceId Rebalance id.
- * @param next Runnable responsible for cache rebalancing start.
- * @param forcedRebFut Rebalance future.
+ * @param forcePreload {@code True} if preload requested by {@link ForceRebalanceExchangeTask}.
+ * @param rebalanceId Rebalance id created by exchange thread.
+ * @param next Runnable responsible for cache rebalancing chain.
+ * @param forcedRebFut External future for forced rebalance.
* @return Rebalancing runnable.
*/
public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
@@ -114,7 +122,6 @@ public interface GridCachePreloader {
* Future result is {@code false} in case rebalancing cancelled or finished with missed partitions and will be
* restarted at current or pending topology.
*
- * Note that topology change creates new futures and finishes previous.
*/
public IgniteInternalFuture<Boolean> rebalanceFuture();
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index af91679..c5e4a81 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -152,6 +152,12 @@ public class GridCachePreloaderAdapter implements GridCachePreloader {
}
/** {@inheritDoc} */
+ @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
+ GridDhtPartitionsExchangeFuture exchFut) {
+ return true;
+ }
+
+ /** {@inheritDoc} */
@Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
GridDhtPartitionsExchangeFuture exchFut) {
return null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 1eeebae..54d3c93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -235,12 +235,12 @@ public class GridDhtPartitionDemander {
/**
* @param fut Future.
- * @return {@code True} if topology changed.
+ * @return {@code True} if rebalance topology version changed by exchange thread or force
+ * reassing exchange occurs, see {@link RebalanceReassignExchangeTask} for details.
*/
private boolean topologyChanged(RebalanceFuture fut) {
- return
- !grp.affinity().lastVersion().equals(fut.topologyVersion()) || // Topology already changed.
- fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
+ return !ctx.exchange().rebalanceTopologyVersion().equals(fut.topVer) ||
+ fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
}
/**
@@ -253,14 +253,21 @@ public class GridDhtPartitionDemander {
}
/**
- * Initiates new rebalance process from given {@code assignments}.
- * If previous rebalance is not finished method cancels it.
- * In case of delayed rebalance method schedules new with configured delay.
+ * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
+ */
+ Collection<UUID> remainingNodes() {
+ return rebalanceFut.remainingNodes();
+ }
+
+ /**
+ * This method initiates new rebalance process from given {@code assignments} by creating new rebalance
+ * future based on them. Cancels previous rebalance future and sends rebalance started event.
+ * In case of delayed rebalance method schedules the new one with configured delay based on {@code lastExchangeFut}.
*
- * @param assignments Assignments.
- * @param force {@code True} if dummy reassign.
- * @param rebalanceId Rebalance id.
- * @param next Runnable responsible for cache rebalancing start.
+ * @param assignments Assignments to process.
+ * @param force {@code True} if preload request by {@link ForceRebalanceExchangeTask}.
+ * @param rebalanceId Rebalance id generated from exchange thread.
+ * @param next Runnable responsible for cache rebalancing chain.
* @param forcedRebFut External future for forced rebalance.
* @return Rebalancing runnable.
*/
@@ -440,17 +447,7 @@ public class GridDhtPartitionDemander {
if (fut.isDone())
return;
- // Must add all remaining node before send first request, for avoid race between add remaining node and
- // processing response, see checkIsDone(boolean).
- for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assignments.entrySet()) {
- UUID nodeId = e.getKey().id();
-
- IgniteDhtDemandedPartitionsMap parts = e.getValue().partitions();
-
- assert parts != null : "Partitions are null [grp=" + grp.cacheOrGroupName() + ", fromNode=" + nodeId + "]";
-
- fut.remaining.put(nodeId, new T2<>(U.currentTimeMillis(), parts));
- }
+ fut.remaining.forEach((key, value) -> value.set1(U.currentTimeMillis()));
}
final CacheConfiguration cfg = grp.config();
@@ -979,6 +976,13 @@ public class GridDhtPartitionDemander {
exchId = assignments.exchangeId();
topVer = assignments.topologyVersion();
+ assignments.forEach((k, v) -> {
+ assert v.partitions() != null :
+ "Partitions are null [grp=" + grp.cacheOrGroupName() + ", fromNode=" + k.id() + "]";
+
+ remaining.put(k.id(), new T2<>(U.currentTimeMillis(), v.partitions()));
+ });
+
this.grp = grp;
this.log = log;
this.rebalanceId = rebalanceId;
@@ -1218,6 +1222,13 @@ public class GridDhtPartitionDemander {
}
/**
+ * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
+ */
+ private synchronized Collection<UUID> remainingNodes() {
+ return remaining.keySet();
+ }
+
+ /**
*
*/
private void sendRebalanceStartedEvent() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 4946d7e..ea7f4c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -17,12 +17,14 @@
package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.stream.Collectors;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cluster.ClusterNode;
@@ -118,19 +120,20 @@ class GridDhtPartitionSupplier {
}
/**
- * Handles new topology version and clears supply context map of outdated contexts.
- *
- * @param topVer Topology version.
+ * Handle topology change and clear supply context map of outdated contexts.
*/
- @SuppressWarnings("ConstantConditions")
- void onTopologyChanged(AffinityTopologyVersion topVer) {
+ void onTopologyChanged() {
synchronized (scMap) {
Iterator<T3<UUID, Integer, AffinityTopologyVersion>> it = scMap.keySet().iterator();
+ Collection<UUID> aliveNodes = grp.shared().discovery().aliveServerNodes().stream()
+ .map(ClusterNode::id)
+ .collect(Collectors.toList());
+
while (it.hasNext()) {
T3<UUID, Integer, AffinityTopologyVersion> t = it.next();
- if (topVer.compareTo(t.get3()) > 0) { // Clear all obsolete contexts.
+ if (!aliveNodes.contains(t.get1())) { // Clear all obsolete contexts.
clearContext(scMap.get(t), log);
it.remove();
@@ -171,17 +174,6 @@ class GridDhtPartitionSupplier {
AffinityTopologyVersion curTop = grp.affinity().lastVersion();
AffinityTopologyVersion demTop = d.topologyVersion();
- if (curTop.compareTo(demTop) > 0) {
- if (log.isDebugEnabled())
- log.debug("Demand request outdated [grp=" + grp.cacheOrGroupName()
- + ", currentTopVer=" + curTop
- + ", demandTopVer=" + demTop
- + ", from=" + nodeId
- + ", topicId=" + topicId + "]");
-
- return;
- }
-
T3<UUID, Integer, AffinityTopologyVersion> contextId = new T3<>(nodeId, topicId, demTop);
if (d.rebalanceId() < 0) { // Demand node requested context cleanup.
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 77f4866..7cf55a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.UUID;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.IgniteInternalFuture;
@@ -39,7 +40,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -160,12 +160,68 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter {
/** {@inheritDoc} */
@Override public void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
- supplier.onTopologyChanged(lastFut.initialVersion());
+ supplier.onTopologyChanged();
demander.onTopologyChanged(lastFut);
}
/** {@inheritDoc} */
+ @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
+ GridDhtPartitionsExchangeFuture exchFut) {
+ if (ctx.kernalContext().clientNode() || rebTopVer.equals(AffinityTopologyVersion.NONE))
+ return false; // No-op.
+
+ if (exchFut.localJoinExchange())
+ return true; // Required, can have outdated updSeq partition counter if node reconnects.
+
+ if (!grp.affinity().cachedVersions().contains(rebTopVer)) {
+ assert rebTopVer.compareTo(grp.localStartVersion()) <= 0 :
+ "Empty hisroty allowed only for newly started cache group [rebTopVer=" + rebTopVer +
+ ", localStartTopVer=" + grp.localStartVersion() + ']';
+
+ return true; // Required, since no history info available.
+ }
+
+ final IgniteInternalFuture<Boolean> rebFut = rebalanceFuture();
+
+ if (rebFut.isDone() && !rebFut.result())
+ return true; // Required, previous rebalance cancelled.
+
+ final AffinityTopologyVersion exchTopVer = exchFut.context().events().topologyVersion();
+
+ Collection<UUID> aliveNodes = ctx.discovery().aliveServerNodes().stream()
+ .map(ClusterNode::id)
+ .collect(Collectors.toList());
+
+ return assignmentsChanged(rebTopVer, exchTopVer) ||
+ !aliveNodes.containsAll(demander.remainingNodes()); // Some of nodes left before rabalance compelete.
+ }
+
+ /**
+ * @param oldTopVer Previous topology version.
+ * @param newTopVer New topology version to check result.
+ * @return {@code True} if affinity assignments changed between two versions for local node.
+ */
+ private boolean assignmentsChanged(AffinityTopologyVersion oldTopVer, AffinityTopologyVersion newTopVer) {
+ final AffinityAssignment aff = grp.affinity().readyAffinity(newTopVer);
+
+ // We should get affinity assignments based on previous rebalance to calculate difference.
+ // Whole history size described by IGNITE_AFFINITY_HISTORY_SIZE constant.
+ final AffinityAssignment prevAff = grp.affinity().cachedVersions().contains(oldTopVer) ?
+ grp.affinity().cachedAffinity(oldTopVer) : null;
+
+ if (prevAff == null)
+ return false;
+
+ boolean assignsChanged = false;
+
+ for (int p = 0; !assignsChanged && p < grp.affinity().partitions(); p++)
+ assignsChanged |= aff.get(p).contains(ctx.localNode()) != prevAff.get(p).contains(ctx.localNode());
+
+ return assignsChanged;
+ }
+
+ /** {@inheritDoc} */
@Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId, GridDhtPartitionsExchangeFuture exchFut) {
assert exchFut == null || exchFut.isDone();
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
index 41dd076..6e847bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
@@ -20,7 +20,7 @@ package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopologyImpl;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -73,9 +73,9 @@ public class GridDhtPreloaderAssignments extends ConcurrentHashMap<ClusterNode,
}
/**
- * @return Topology version.
+ * @return Topology version based on last {@link GridDhtPartitionTopologyImpl#readyTopVer}.
*/
- AffinityTopologyVersion topologyVersion() {
+ public AffinityTopologyVersion topologyVersion() {
return topVer;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
index 5653177..46b09ac 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
@@ -149,7 +149,6 @@ public class ClusterBaselineNodesMetricsSelfTest extends GridCommonAbstractTest
private void resetBlt() throws Exception {
resetBaselineTopology();
- waitForRebalancing();
awaitPartitionMapExchange();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
index ba3ad5a..4a950dd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
@@ -98,14 +98,14 @@ public class CacheValidatorMetricsTest extends GridCommonAbstractTest implements
startGrid(2);
- waitForRebalancing();
+ awaitPartitionMapExchange();
assertCacheStatus(CACHE_NAME_1, true, true);
assertCacheStatus(CACHE_NAME_2, true, true);
stopGrid(1);
- waitForRebalancing();
+ awaitPartitionMapExchange();
// Invalid for writing due to invalid topology.
assertCacheStatus(CACHE_NAME_1, true, false);
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
index 83eff89..23ba4b3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
@@ -22,29 +22,23 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.cache.CacheRebalanceMode;
import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.events.CacheRebalancingEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -58,9 +52,6 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.configuration.CacheConfiguration.DFLT_REBALANCE_BATCH_SIZE;
import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS;
import static org.apache.ignite.events.EventType.EVTS_CACHE_REBALANCE;
-import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STOPPED;
-import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
-import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
@@ -142,15 +133,6 @@ public class GridCacheDhtPreloadSelfTest extends GridCommonAbstractTest {
}
/** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
-// resetLog4j(Level.DEBUG, true,
-// // Categories.
-// GridDhtPreloader.class.getPackage().getName(),
-// GridDhtPartitionTopologyImpl.class.getName(),
-// GridDhtLocalPartition.class.getName());
- }
-
- /** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
backups = DFLT_BACKUPS;
partitions = DFLT_PARTITIONS;
@@ -227,11 +209,6 @@ public class GridCacheDhtPreloadSelfTest extends GridCommonAbstractTest {
*/
private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
throws Exception {
-// resetLog4j(Level.DEBUG, true,
-// // Categories.
-// GridDhtPreloader.class.getPackage().getName(),
-// GridDhtPartitionTopologyImpl.class.getName(),
-// GridDhtLocalPartition.class.getName());
try {
Ignite ignite1 = startGrid(0);
@@ -270,8 +247,6 @@ public class GridCacheDhtPreloadSelfTest extends GridCommonAbstractTest {
info(">>> Finished checking nodes [keyCnt=" + keyCnt + ", nodeCnt=" + nodeCnt + ", grids=" +
U.grids2names(ignites) + ']');
- Collection<IgniteFuture<?>> futs = new LinkedList<>();
-
Ignite last = F.last(ignites);
for (Iterator<Ignite> it = ignites.iterator(); it.hasNext(); ) {
@@ -285,21 +260,8 @@ public class GridCacheDhtPreloadSelfTest extends GridCommonAbstractTest {
checkActiveState(ignites);
- final UUID nodeId = g.cluster().localNode().id();
-
it.remove();
- futs.add(waitForLocalEvent(last.events(), new P1<Event>() {
- @Override public boolean apply(Event e) {
- CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
-
- ClusterNode node = evt.discoveryNode();
-
- return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
- (evt.discoveryEventType() == EVT_NODE_LEFT || evt.discoveryEventType() == EVT_NODE_FAILED);
- }
- }, EVT_CACHE_REBALANCE_STOPPED));
-
info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
stopGrid(g.name());
@@ -312,14 +274,6 @@ public class GridCacheDhtPreloadSelfTest extends GridCommonAbstractTest {
awaitPartitionMapExchange(); // Need wait, otherwise test logic is broken if EVT_NODE_FAILED exchanges are merged.
}
- info("Waiting for preload futures: " + F.view(futs, new IgnitePredicate<IgniteFuture<?>>() {
- @Override public boolean apply(IgniteFuture<?> fut) {
- return !fut.isDone();
- }
- }));
-
- X.waitAll(futs);
-
info("Finished waiting for preload futures.");
assert last != null;
@@ -499,11 +453,6 @@ public class GridCacheDhtPreloadSelfTest extends GridCommonAbstractTest {
*/
private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
throws Exception {
-// resetLog4j(Level.DEBUG, true,
-// // Categories.
-// GridDhtPreloader.class.getPackage().getName(),
-// GridDhtPartitionTopologyImpl.class.getName(),
-// GridDhtLocalPartition.class.getName());
try {
Ignite ignite1 = startGrid(0);
@@ -555,28 +504,13 @@ public class GridCacheDhtPreloadSelfTest extends GridCommonAbstractTest {
it.remove();
- Collection<IgniteFuture<?>> futs = new LinkedList<>();
-
- for (Ignite gg : ignites)
- futs.add(waitForLocalEvent(gg.events(), new P1<Event>() {
- @Override public boolean apply(Event e) {
- CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
-
- ClusterNode node = evt.discoveryNode();
-
- return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
- (evt.discoveryEventType() == EVT_NODE_LEFT ||
- evt.discoveryEventType() == EVT_NODE_FAILED);
- }
- }, EVT_CACHE_REBALANCE_STOPPED));
-
info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
stopGrid(g.name());
info(">>> Waiting for preload futures [leftNode=" + g.name() + ", remaining=" + U.grids2names(ignites) + ']');
- X.waitAll(futs);
+ awaitPartitionMapExchange();
info("After grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java
index 20f292b..14c8571 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java
@@ -807,9 +807,6 @@ public class IgniteCacheAtomicProtocolTest extends GridCommonAbstractTest {
startServers(2);
// Waiting for minor topology changing because of late affinity assignment.
- waitForRebalancing(0, 2, 1);
- waitForRebalancing(1, 2, 1);
-
awaitPartitionMapExchange();
Ignite srv0 = ignite(0);
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
index 4ebcd5d..0a8698a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
@@ -17,10 +17,11 @@
package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
-import org.apache.ignite.Ignite;
+import java.util.Collections;
import org.apache.ignite.cache.CacheRebalanceMode;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
@@ -43,7 +44,7 @@ public class GridCacheRebalancingAsyncSelfTest extends GridCacheRebalancingSyncS
* @throws Exception Exception.
*/
public void testNodeFailedAtRebalancing() throws Exception {
- Ignite ignite = startGrid(0);
+ IgniteEx ignite = startGrid(0);
generateData(ignite, 0, 0);
@@ -60,7 +61,7 @@ public class GridCacheRebalancingAsyncSelfTest extends GridCacheRebalancingSyncS
((TestTcpDiscoverySpi)grid(1).configuration().getDiscoverySpi()).simulateNodeFailure();
- waitForRebalancing(0, 3);
+ awaitPartitionMapExchange(false, false, Collections.singletonList(ignite.localNode()));
checkSupplyContextMapIsEmpty();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
new file mode 100644
index 0000000..3965290
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test cases for checking cancellation rebalancing process if some events occurs.
+ */
+public class GridCacheRebalancingCancelTest extends GridCommonAbstractTest {
+ /** */
+ private static final String DHT_PARTITIONED_CACHE = "cacheP";
+
+ /** */
+ private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration dfltCfg = super.getConfiguration(igniteInstanceName);
+
+ ((TcpDiscoverySpi)dfltCfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+ dfltCfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+ return dfltCfg;
+ }
+
+ /**
+ * Test rebalance not cancelled when client node join to cluster.
+ *
+ * @throws Exception Exception.
+ */
+ public void testClientNodeJoinAtRebalancing() throws Exception {
+ final IgniteEx ignite0 = startGrid(0);
+
+ IgniteCache<Integer, Integer> cache = ignite0.createCache(
+ new CacheConfiguration<Integer, Integer>(DHT_PARTITIONED_CACHE)
+ .setCacheMode(CacheMode.PARTITIONED)
+ .setRebalanceMode(CacheRebalanceMode.ASYNC)
+ .setBackups(1)
+ .setRebalanceOrder(2)
+ .setAffinity(new RendezvousAffinityFunction(false)));
+
+ for (int i = 0; i < 2048; i++)
+ cache.put(i, i);
+
+ TestRecordingCommunicationSpi.spi(ignite0)
+ .blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+ @Override public boolean apply(ClusterNode node, Message msg) {
+ return (msg instanceof GridDhtPartitionSupplyMessage)
+ && ((GridCacheGroupIdMessage)msg).groupId() == groupIdForCache(ignite0, DHT_PARTITIONED_CACHE);
+ }
+ });
+
+ final IgniteEx ignite1 = startGrid(1);
+
+ TestRecordingCommunicationSpi.spi(ignite0).waitForBlocked();
+
+ GridDhtPartitionDemander.RebalanceFuture fut = (GridDhtPartitionDemander.RebalanceFuture)ignite1.context().
+ cache().internalCache(DHT_PARTITIONED_CACHE).preloader().rebalanceFuture();
+
+ String igniteClntName = getTestIgniteInstanceName(2);
+
+ startGrid(igniteClntName, optimize(getConfiguration(igniteClntName).setClientMode(true)));
+
+ // Resend delayed rebalance messages.
+ TestRecordingCommunicationSpi.spi(ignite0).stopBlock(true);
+
+ awaitPartitionMapExchange();
+
+ // Previous rebalance future should not be cancelled.
+ assertTrue(fut.result());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
index 1280e87..cb414ed 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
@@ -141,7 +141,8 @@ public class GridCacheRebalancingPartitionCountersTest extends GridCommonAbstrac
assertTrue(primaryRemoved);
ignite.cluster().active(true);
- waitForRebalancing();
+
+ awaitPartitionMapExchange();
List<String> issues = new ArrayList<>();
HashMap<Integer, Long> partMap = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
index ed51cf3..a027a41 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
@@ -21,10 +21,9 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.CacheRebalanceMode;
import org.apache.ignite.cluster.ClusterNode;
@@ -42,11 +41,13 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartit
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.PA;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.resources.LoggerResource;
import org.apache.ignite.spi.IgniteSpiException;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -68,6 +69,9 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
/** */
private static final int TEST_SIZE = 100_000;
+ /** */
+ private static final long TOPOLOGY_STILLNESS_TIME = 30_000L;
+
/** partitioned cache name. */
protected static final String CACHE_NAME_DHT_PARTITIONED = "cacheP";
@@ -89,11 +93,11 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
/** */
private volatile boolean concurrentStartFinished3;
- /** */
- private volatile boolean record;
-
- /** */
- private final ConcurrentHashMap<Class, AtomicInteger> map = new ConcurrentHashMap<>();
+ /**
+ * Time in milliseconds of last received {@link GridDhtPartitionsSingleMessage}
+ * or {@link GridDhtPartitionsFullMessage} using {@link CollectingCommunicationSpi}.
+ */
+ private static volatile long lastPartMsgTime;
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -102,7 +106,7 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setIpFinder(ipFinder);
((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setForceServerMode(true);
- TcpCommunicationSpi commSpi = new CountingCommunicationSpi();
+ TcpCommunicationSpi commSpi = new CollectingCommunicationSpi();
commSpi.setLocalPort(GridTestUtils.getNextCommPort(getClass()));
commSpi.setTcpNoDelay(true);
@@ -232,47 +236,35 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
startGrid(1);
- int waitMinorVer = ignite.configuration().isLateAffinityAssignment() ? 1 : 0;
-
- waitForRebalancing(0, 2, waitMinorVer);
- waitForRebalancing(1, 2, waitMinorVer);
-
awaitPartitionMapExchange(true, true, null, true);
checkPartitionMapExchangeFinished();
- checkPartitionMapMessagesAbsent();
+ awaitPartitionMessagesAbsent();
stopGrid(0);
- waitForRebalancing(1, 3);
-
awaitPartitionMapExchange(true, true, null, true);
checkPartitionMapExchangeFinished();
- checkPartitionMapMessagesAbsent();
+ awaitPartitionMessagesAbsent();
startGrid(2);
- waitForRebalancing(1, 4, waitMinorVer);
- waitForRebalancing(2, 4, waitMinorVer);
-
awaitPartitionMapExchange(true, true, null, true);
checkPartitionMapExchangeFinished();
- checkPartitionMapMessagesAbsent();
+ awaitPartitionMessagesAbsent();
stopGrid(2);
- waitForRebalancing(1, 5);
-
awaitPartitionMapExchange(true, true, null, true);
checkPartitionMapExchangeFinished();
- checkPartitionMapMessagesAbsent();
+ awaitPartitionMessagesAbsent();
long spend = (System.currentTimeMillis() - start) / 1000;
@@ -331,13 +323,10 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
startGrid(4);
- waitForRebalancing(3, 6);
- waitForRebalancing(4, 6);
+ awaitPartitionMapExchange(true, true, null);
concurrentStartFinished = true;
- awaitPartitionMapExchange(true, true, null);
-
checkSupplyContextMapIsEmpty();
t1.join();
@@ -442,27 +431,29 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
}
/**
+ * Method checks for {@link GridDhtPartitionsSingleMessage} or {@link GridDhtPartitionsFullMessage}
+ * not received within {@code TOPOLOGY_STILLNESS_TIME} bound.
+ *
* @throws Exception If failed.
*/
- protected void checkPartitionMapMessagesAbsent() throws Exception {
- map.clear();
-
- record = true;
-
- log.info("Checking GridDhtPartitions*Message absent (it will take 30 SECONDS) ... ");
-
- U.sleep(30_000);
-
- record = false;
-
- AtomicInteger iF = map.get(GridDhtPartitionsFullMessage.class);
- AtomicInteger iS = map.get(GridDhtPartitionsSingleMessage.class);
-
- Integer fullMap = iF != null ? iF.get() : null;
- Integer singleMap = iS != null ? iS.get() : null;
-
- assertTrue("Unexpected full map messages: " + fullMap, fullMap == null || fullMap.equals(1)); // 1 message can be sent right after all checks passed.
- assertNull("Unexpected single map messages", singleMap);
+ protected void awaitPartitionMessagesAbsent() throws Exception {
+ log.info("Checking GridDhtPartitions*Message absent (it will take up to " +
+ TOPOLOGY_STILLNESS_TIME + " ms) ... ");
+
+ // Start waiting new messages from current point of time.
+ lastPartMsgTime = U.currentTimeMillis();
+
+ assertTrue("Should not have partition Single or Full messages within bound " +
+ TOPOLOGY_STILLNESS_TIME + " ms.",
+ GridTestUtils.waitForCondition(
+ new GridAbsPredicateX() {
+ @Override public boolean applyx() {
+ return lastPartMsgTime + TOPOLOGY_STILLNESS_TIME < U.currentTimeMillis();
+ }
+ },
+ 2 * TOPOLOGY_STILLNESS_TIME // 30 sec to gain stable topology and 30 sec of silence.
+ )
+ );
}
/** {@inheritDoc} */
@@ -495,11 +486,7 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
while (!concurrentStartFinished2)
U.sleep(10);
- waitForRebalancing(0, 5, 0);
- waitForRebalancing(1, 5, 0);
- waitForRebalancing(2, 5, 0);
- waitForRebalancing(3, 5, 0);
- waitForRebalancing(4, 5, 0);
+ awaitPartitionMapExchange();
//New cache should start rebalancing.
CacheConfiguration<Integer, Integer> cacheRCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
@@ -552,12 +539,6 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
t2.join();
t3.join();
- waitForRebalancing(0, 5, 1);
- waitForRebalancing(1, 5, 1);
- waitForRebalancing(2, 5, 1);
- waitForRebalancing(3, 5, 1);
- waitForRebalancing(4, 5, 1);
-
awaitPartitionMapExchange(true, true, null);
checkSupplyContextMapIsEmpty();
@@ -577,35 +558,23 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
stopGrid(1);
- waitForRebalancing(0, 6);
- waitForRebalancing(2, 6);
- waitForRebalancing(3, 6);
- waitForRebalancing(4, 6);
-
awaitPartitionMapExchange(true, true, null);
checkSupplyContextMapIsEmpty();
stopGrid(0);
- waitForRebalancing(2, 7);
- waitForRebalancing(3, 7);
- waitForRebalancing(4, 7);
-
awaitPartitionMapExchange(true, true, null);
checkSupplyContextMapIsEmpty();
stopGrid(2);
- waitForRebalancing(3, 8);
- waitForRebalancing(4, 8);
-
awaitPartitionMapExchange(true, true, null);
checkPartitionMapExchangeFinished();
- checkPartitionMapMessagesAbsent();
+ awaitPartitionMessagesAbsent();
checkSupplyContextMapIsEmpty();
@@ -613,7 +582,7 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
stopGrid(3);
- waitForRebalancing(4, 9);
+ awaitPartitionMapExchange();
checkSupplyContextMapIsEmpty();
@@ -634,36 +603,26 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
/**
*
*/
- private class CountingCommunicationSpi extends TcpCommunicationSpi {
+ private static class CollectingCommunicationSpi extends TcpCommunicationSpi {
+ /** */
+ @LoggerResource
+ private IgniteLogger log;
+
/** {@inheritDoc} */
@Override public void sendMessage(final ClusterNode node, final Message msg,
final IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
final Object msg0 = ((GridIoMessage)msg).message();
- recordMessage(msg0);
+ if (msg0 instanceof GridDhtPartitionsSingleMessage ||
+ msg0 instanceof GridDhtPartitionsFullMessage) {
+ lastPartMsgTime = U.currentTimeMillis();
- super.sendMessage(node, msg, ackC);
- }
-
- /**
- * @param msg Message.
- */
- private void recordMessage(Object msg) {
- if (record) {
- Class id = msg.getClass();
-
- AtomicInteger ai = map.get(id);
-
- if (ai == null) {
- ai = new AtomicInteger();
-
- AtomicInteger oldAi = map.putIfAbsent(id, ai);
-
- (oldAi != null ? oldAi : ai).incrementAndGet();
- }
- else
- ai.incrementAndGet();
+ log.info("Last seen time of GridDhtPartitionsSingleMessage or GridDhtPartitionsFullMessage updated " +
+ "[lastPartMsgTime=" + lastPartMsgTime +
+ ", node=" + node.id() + ']');
}
+
+ super.sendMessage(node, msg, ackC);
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
index 7e9765c..13a98e4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
@@ -383,7 +383,7 @@ public class ClientAffinityAssignmentWithBaselineTest extends GridCommonAbstract
startGrid("flaky");
System.out.println("### Starting rebalancing after flaky node join");
- waitForRebalancing();
+ awaitPartitionMapExchange();
System.out.println("### Rebalancing is finished after flaky node join");
awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
@@ -689,7 +689,7 @@ public class ClientAffinityAssignmentWithBaselineTest extends GridCommonAbstract
ig0.cluster().setBaselineTopology(fullBlt.subList(0, newBaselineSize));
System.out.println("### Starting rebalancing after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
- waitForRebalancing();
+ awaitPartitionMapExchange();
System.out.println("### Rebalancing is finished after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
index 8f2e738..3500c8d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
@@ -157,7 +157,8 @@ public class SlowHistoricalRebalanceSmallHistoryTest extends GridCommonAbstractT
SUPPLY_MESSAGE_LATCH.get().countDown();
- waitForRebalancing(); // Partition is OWNING on grid(0) and grid(1)
+ // Partition is OWNING on grid(0) and grid(1)
+ awaitPartitionMapExchange();
for (int i = 0; i < 2; i++) {
for (int j = 0; i < 500; i++)
@@ -178,7 +179,7 @@ public class SlowHistoricalRebalanceSmallHistoryTest extends GridCommonAbstractT
startGrid(0);
- waitForRebalancing();
+ awaitPartitionMapExchange();
assertEquals(2, grid(1).context().discovery().aliveServerNodes().size());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index 4d26823..a28ec5f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -194,7 +194,7 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
grid.cluster().setBaselineTopology(grid.cluster().topologyVersion());
- waitForRebalancing();
+ awaitPartitionMapExchange();
}
catch (Throwable expected) {
// There can be any exception. Do nothing.
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
index 78f3c03..9de2702 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
@@ -120,7 +120,8 @@ public class GridMarshallerMappingConsistencyTest extends GridCommonAbstractTest
c1.put(k, new DummyObject(k));
startGrid(2);
- waitForRebalancing();
+
+ awaitPartitionMapExchange();
stopAllGrids();
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 2c5091c..313cd71 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -688,32 +688,34 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
if (affNodesCnt != ownerNodesCnt || !affNodes.containsAll(owners) ||
(waitEvicts && loc != null && loc.state() != GridDhtPartitionState.OWNING)) {
+ if (i % 50 == 0)
+ LT.warn(log(), "Waiting for topology map update [" +
+ "igniteInstanceName=" + g.name() +
+ ", cache=" + cfg.getName() +
+ ", cacheId=" + dht.context().cacheId() +
+ ", topVer=" + top.readyTopologyVersion() +
+ ", p=" + p +
+ ", affNodesCnt=" + affNodesCnt +
+ ", ownersCnt=" + ownerNodesCnt +
+ ", affNodes=" + F.nodeIds(affNodes) +
+ ", owners=" + F.nodeIds(owners) +
+ ", topFut=" + topFut +
+ ", locNode=" + g.cluster().localNode() + ']');
+ }
+ else
+ match = true;
+ }
+ else {
+ if (i % 50 == 0)
LT.warn(log(), "Waiting for topology map update [" +
"igniteInstanceName=" + g.name() +
", cache=" + cfg.getName() +
", cacheId=" + dht.context().cacheId() +
", topVer=" + top.readyTopologyVersion() +
+ ", started=" + dht.context().started() +
", p=" + p +
- ", affNodesCnt=" + affNodesCnt +
- ", ownersCnt=" + ownerNodesCnt +
- ", affNodes=" + F.nodeIds(affNodes) +
- ", owners=" + F.nodeIds(owners) +
- ", topFut=" + topFut +
+ ", readVer=" + readyVer +
", locNode=" + g.cluster().localNode() + ']');
- }
- else
- match = true;
- }
- else {
- LT.warn(log(), "Waiting for topology map update [" +
- "igniteInstanceName=" + g.name() +
- ", cache=" + cfg.getName() +
- ", cacheId=" + dht.context().cacheId() +
- ", topVer=" + top.readyTopologyVersion() +
- ", started=" + dht.context().started() +
- ", p=" + p +
- ", readVer=" + readyVer +
- ", locNode=" + g.cluster().localNode() + ']');
}
if (!match) {
@@ -998,81 +1000,6 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
}
/**
- * @param id Node id.
- * @param major Major ver.
- * @param minor Minor ver.
- * @throws IgniteCheckedException If failed.
- */
- protected void waitForRebalancing(int id, int major, int minor) throws IgniteCheckedException {
- waitForRebalancing(grid(id), new AffinityTopologyVersion(major, minor));
- }
-
- /**
- * @param id Node id.
- * @param major Major ver.
- * @throws IgniteCheckedException If failed.
- */
- protected void waitForRebalancing(int id, int major) throws IgniteCheckedException {
- waitForRebalancing(grid(id), new AffinityTopologyVersion(major));
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- protected void waitForRebalancing() throws IgniteCheckedException {
- for (Ignite ignite : G.allGrids())
- waitForRebalancing((IgniteEx)ignite, null);
- }
-
- /**
- * @param ignite Node.
- * @param top Topology version.
- * @throws IgniteCheckedException If failed.
- */
- protected void waitForRebalancing(IgniteEx ignite, AffinityTopologyVersion top) throws IgniteCheckedException {
- if (ignite.configuration().isClientMode())
- return;
-
- boolean finished = false;
-
- long stopTime = System.currentTimeMillis() + 60_000;
-
- while (!finished && (System.currentTimeMillis() < stopTime)) {
- finished = true;
-
- if (top == null)
- top = ignite.context().discovery().topologyVersionEx();
-
- for (GridCacheAdapter c : ignite.context().cache().internalCaches()) {
- GridDhtPartitionDemander.RebalanceFuture fut =
- (GridDhtPartitionDemander.RebalanceFuture)c.preloader().rebalanceFuture();
-
- if (fut.topologyVersion() == null || fut.topologyVersion().compareTo(top) < 0) {
- finished = false;
-
- log.info("Unexpected future version, will retry [futVer=" + fut.topologyVersion() +
- ", expVer=" + top + ']');
-
- U.sleep(100);
-
- break;
- }
- else if (!fut.get()) {
- finished = false;
-
- log.warning("Rebalancing finished with missed partitions.");
-
- U.sleep(100);
-
- break;
- }
- }
- }
-
- assertTrue(finished);
- }
-
- /**
* @param ignite Node.
*/
public void dumpCacheDebugInfo(Ignite ignite) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/137dd06a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index 55436d6..5e94052 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -53,6 +53,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePut
import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxReentryNearSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRabalancingDelayedPartitionMapExchangeSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingAsyncSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingCancelTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest;
@@ -152,6 +153,7 @@ public class IgniteCacheTestSuite3 extends TestSuite {
suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class);
suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
+ suite.addTestSuite(GridCacheRebalancingCancelTest.class);
// Test for byte array value special case.
suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);
[45/50] [abbrv] ignite git commit: IGNITE-8869
PartitionsExchangeOnDiscoveryHistoryOverflowTest hangs on TeamCity. - Fixes
#4277.
Posted by av...@apache.org.
IGNITE-8869 PartitionsExchangeOnDiscoveryHistoryOverflowTest hangs on TeamCity. - Fixes #4277.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b718e443
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b718e443
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b718e443
Branch: refs/heads/ignite-8446
Commit: b718e4438e469ad0341f1bf9a605c2fe3ab5970c
Parents: 24bd54a
Author: Ivan Daschinskiy <iv...@gmail.com>
Authored: Thu Aug 2 18:57:22 2018 +0300
Committer: Dmitriy Pavlov <dp...@apache.org>
Committed: Thu Aug 2 18:57:22 2018 +0300
----------------------------------------------------------------------
.../dht/preloader/latch/ExchangeLatchManager.java | 12 ++++++------
...artitionsExchangeOnDiscoveryHistoryOverflowTest.java | 2 --
2 files changed, 6 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b718e443/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
index 8a57b90..0798d33 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -38,13 +38,13 @@ import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.GridTopic;
import org.apache.ignite.internal.managers.communication.GridIoManager;
import org.apache.ignite.internal.managers.communication.GridIoPolicy;
-import org.apache.ignite.internal.managers.discovery.DiscoCache;
import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.lang.IgniteProductVersion;
import org.jetbrains.annotations.Nullable;
@@ -228,13 +228,13 @@ public class ExchangeLatchManager {
if (topVer == AffinityTopologyVersion.NONE)
return discovery.aliveServerNodes();
else {
- DiscoCache discoCache = discovery.discoCache(topVer);
+ Collection<ClusterNode> histNodes = discovery.topology(topVer.topologyVersion());
- if (discoCache != null)
- return discoCache.aliveServerNodes();
+ if (histNodes != null)
+ return histNodes.stream().filter(n -> !CU.clientNode(n) && !n.isDaemon() && discovery.alive(n))
+ .collect(Collectors.toList());
else
- throw new IgniteException("DiscoCache not found for topology "
- + topVer
+ throw new IgniteException("Topology " + topVer + " not found in discovery history "
+ "; consider increasing IGNITE_DISCOVERY_HISTORY_SIZE property. Current value is "
+ IgniteSystemProperties.getInteger(IgniteSystemProperties.IGNITE_DISCOVERY_HISTORY_SIZE, -1));
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b718e443/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeOnDiscoveryHistoryOverflowTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeOnDiscoveryHistoryOverflowTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeOnDiscoveryHistoryOverflowTest.java
index f9d6bf0..c0896c8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeOnDiscoveryHistoryOverflowTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeOnDiscoveryHistoryOverflowTest.java
@@ -122,8 +122,6 @@ public class PartitionsExchangeOnDiscoveryHistoryOverflowTest extends IgniteCach
* @throws Exception In case of error.
*/
public void testDynamicCacheCreation() throws Exception {
- fail("https://issues.apache.org/jira/browse/IGNITE-8869 should fail if ignite-8869 reverts");
-
for (int iter = 0; iter < 5; iter++) {
log.info("Iteration: " + iter);
[11/50] [abbrv] ignite git commit: IGNITE-9139 Web Console: Fixed
issue with dropdown for chart options on Queries screen.
Posted by av...@apache.org.
IGNITE-9139 Web Console: Fixed issue with dropdown for chart options on Queries screen.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f6c67e86
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f6c67e86
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f6c67e86
Branch: refs/heads/ignite-8446
Commit: f6c67e864212bf66abc38293f7c7c27e380116c2
Parents: ce30b1e
Author: Dmitriy Shabalin <dm...@gmail.com>
Authored: Tue Jul 31 18:12:13 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Jul 31 18:12:13 2018 +0700
----------------------------------------------------------------------
.../app/components/page-queries/index.js | 2 ++
.../app/components/page-queries/style.scss | 20 ++++++++++++++++++++
.../frontend/views/sql/chart-settings.tpl.pug | 2 +-
3 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f6c67e86/modules/web-console/frontend/app/components/page-queries/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/index.js b/modules/web-console/frontend/app/components/page-queries/index.js
index ab64b44..9434dc1 100644
--- a/modules/web-console/frontend/app/components/page-queries/index.js
+++ b/modules/web-console/frontend/app/components/page-queries/index.js
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import './style.scss';
+
import angular from 'angular';
import queriesNotebooksList from './components/queries-notebooks-list';
http://git-wip-us.apache.org/repos/asf/ignite/blob/f6c67e86/modules/web-console/frontend/app/components/page-queries/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/style.scss b/modules/web-console/frontend/app/components/page-queries/style.scss
new file mode 100644
index 0000000..8f13c2e
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-queries/style.scss
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+button.select-toggle.btn-chart-column-agg-fx::after {
+ right: 0;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f6c67e86/modules/web-console/frontend/views/sql/chart-settings.tpl.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/views/sql/chart-settings.tpl.pug b/modules/web-console/frontend/views/sql/chart-settings.tpl.pug
index 5a1ceed..054067f 100644
--- a/modules/web-console/frontend/views/sql/chart-settings.tpl.pug
+++ b/modules/web-console/frontend/views/sql/chart-settings.tpl.pug
@@ -36,5 +36,5 @@
ul.chart-settings-columns-list(dnd-list='paragraph.chartValCols' dnd-drop='chartAcceptValColumn(paragraph, item)')
li(ng-repeat='col in paragraph.chartValCols track by $index')
.btn.btn-default.btn-chart-column(ng-style='chartColor($index)') {{col.label}}
- button.btn-chart-column-agg-fx.select-toggle(ng-change='applyChartSettings(paragraph)' ng-show='paragraphTimeSpanVisible(paragraph)' ng-style='chartColor($index)' ng-model='col.aggFx' placeholder='...' bs-select bs-options='item for item in aggregateFxs' data-container='false' tabindex='-1')
+ button.btn-chart-column-agg-fx.select-toggle(ng-change='applyChartSettings(paragraph)' ng-show='paragraphTimeSpanVisible(paragraph)' ng-style='chartColor($index)' ng-model='col.aggFx' placeholder='...' bs-select bs-options='item for item in aggregateFxs' tabindex='-1')
i.fa.fa-close(ng-click='chartRemoveValColumn(paragraph, $index)')
[38/50] [abbrv] ignite git commit: IGNITE-9151 Web Console: Fixed
logic of removing cluster items: caches, models, igfss.
Posted by av...@apache.org.
IGNITE-9151 Web Console: Fixed logic of removing cluster items: caches, models, igfss.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1cdf7ef9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1cdf7ef9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1cdf7ef9
Branch: refs/heads/ignite-8446
Commit: 1cdf7ef992c36a1b8f52021dd91705c51257d1c1
Parents: 5fc18e6
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Thu Aug 2 17:18:50 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Aug 2 17:18:50 2018 +0700
----------------------------------------------------------------------
.../form-field/showValidationError.directive.js | 6 +-
.../grid-column-selector/controller.js | 8 +-
.../list-editable-save-on-changes/directives.js | 14 ++-
.../list-editable-transclude/directive.js | 12 +-
.../app/components/list-editable/controller.js | 6 +-
.../components/cluster-edit-form/controller.js | 2 -
.../components/model-edit-form/controller.js | 10 +-
.../page-configure-basic/controller.js | 13 ++-
.../pco-grid-column-categories/directive.js | 4 +-
.../components/formUICanExitGuard.js | 12 +-
.../components/modal-import-models/component.js | 23 ++--
.../tables-action-cell/component.js | 14 ++-
.../modal-preview-project/controller.js | 10 +-
.../components/pc-form-field-size/controller.js | 8 +-
.../components/pc-items-table/controller.js | 26 ++++-
.../components/pc-ui-grid-filters/directive.js | 4 +-
.../components/pc-ui-grid-filters/index.js | 5 +-
.../components/pcIsInCollection.js | 4 +-
.../page-configure/components/pcValidation.js | 8 +-
.../app/components/page-configure/index.js | 14 ++-
.../app/components/page-configure/reducer.js | 90 ++++++++++++++-
.../page-configure/services/ConfigureState.js | 5 +
.../app/components/page-configure/states.js | 88 ++++++++------
.../components/page-configure/store/effects.js | 114 +++++++++++--------
.../page-configure/store/selectors.js | 50 +++++++-
.../transitionHooks/errorState.js | 5 +-
.../frontend/app/components/page-signin/run.js | 5 +-
.../components/panel-collapsible/controller.js | 19 +++-
.../app/components/ui-grid-filters/directive.js | 3 +-
.../app/components/ui-grid-filters/index.js | 4 +-
.../generator/ConfigurationGenerator.js | 3 +-
.../field/bs-select-placeholder.directive.js | 4 +-
.../modules/form/validator/unique.directive.js | 6 +-
.../web-console/frontend/app/services/Caches.js | 10 +-
.../frontend/app/services/Clusters.js | 32 ++++--
.../frontend/app/services/FormUtils.service.js | 13 ++-
.../web-console/frontend/app/services/IGFSs.js | 8 +-
.../web-console/frontend/app/services/Models.js | 17 ++-
.../frontend/app/services/exceptionHandler.js | 4 +-
39 files changed, 509 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/form-field/showValidationError.directive.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/form-field/showValidationError.directive.js b/modules/web-console/frontend/app/components/form-field/showValidationError.directive.js
index 32123cc..8720c56 100644
--- a/modules/web-console/frontend/app/components/form-field/showValidationError.directive.js
+++ b/modules/web-console/frontend/app/components/form-field/showValidationError.directive.js
@@ -24,10 +24,14 @@ export function directive($timeout) {
require: ['ngModel', '?^^bsCollapseTarget', '?^^igniteFormField', '?^^panelCollapsible'],
link(scope, el, attr, [ngModel, bsCollapseTarget, igniteFormField, panelCollapsible]) {
const off = scope.$on('$showValidationError', (e, target) => {
- if (target !== ngModel) return;
+ if (target !== ngModel)
+ return;
+
ngModel.$setTouched();
+
bsCollapseTarget && bsCollapseTarget.open();
panelCollapsible && panelCollapsible.open();
+
$timeout(() => {
if (el[0].scrollIntoViewIfNeeded)
el[0].scrollIntoViewIfNeeded();
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/grid-column-selector/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-column-selector/controller.js b/modules/web-console/frontend/app/components/grid-column-selector/controller.js
index 1f04861..2f5554e 100644
--- a/modules/web-console/frontend/app/components/grid-column-selector/controller.js
+++ b/modules/web-console/frontend/app/components/grid-column-selector/controller.js
@@ -77,9 +77,13 @@ export default class GridColumnSelectorController {
}
findScrollToNext(columns, prevColumns) {
- if (!prevColumns) return;
+ if (!prevColumns)
+ return;
+
const diff = difference(columns, prevColumns);
- if (diff.length === 1 && columns.includes(diff[0])) return diff[0];
+
+ if (diff.length === 1 && columns.includes(diff[0]))
+ return diff[0];
}
setSelectedColumns() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/directives.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/directives.js b/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/directives.js
index 4b6e785..f408647 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/directives.js
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/directives.js
@@ -30,7 +30,9 @@ export function ngModel() {
* @param {ng.INgModelController} ngModel
*/
link(scope, el, attr, {ngModel, list}) {
- if (!list) return;
+ if (!list)
+ return;
+
ngModel.$viewChangeListeners.push(() => {
el[0].dispatchEvent(new CustomEvent(CUSTOM_EVENT_TYPE, {bubbles: true, cancelable: true}));
});
@@ -54,15 +56,21 @@ export function listEditableTransclude() {
* @param {ListEditableController} list
*/
link(scope, el, attr, {list, transclude}) {
- if (attr.listEditableTransclude !== 'itemEdit') return;
- if (!list) return;
+ if (attr.listEditableTransclude !== 'itemEdit')
+ return;
+
+ if (!list)
+ return;
+
let listener = (e) => {
e.stopPropagation();
scope.$evalAsync(() => {
if (scope.form.$valid) list.save(scope.item, transclude.$index);
});
};
+
el[0].addEventListener(CUSTOM_EVENT_TYPE, listener);
+
scope.$on('$destroy', () => {
el[0].removeEventListener(CUSTOM_EVENT_TYPE, listener);
listener = null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/directive.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/directive.js b/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/directive.js
index 55b4724..36750ae 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/directive.js
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/directive.js
@@ -63,7 +63,9 @@ export class ListEditableTransclude {
[itemName]: {
get: () => {
// Scope might get destroyed
- if (!this.$scope) return;
+ if (!this.$scope)
+ return;
+
return this.$scope.item;
},
set: (value) => {
@@ -80,7 +82,9 @@ export class ListEditableTransclude {
$form: {
get: () => {
// Scope might get destroyed
- if (!this.$scope) return;
+ if (!this.$scope)
+ return;
+
return this.$scope.form;
},
// Allows to delete property later
@@ -98,7 +102,9 @@ export class ListEditableTransclude {
* @returns {number}
*/
get $index() {
- if (!this.$scope) return;
+ if (!this.$scope)
+ return;
+
return this.$scope.$index;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/list-editable/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/list-editable/controller.js b/modules/web-console/frontend/app/components/list-editable/controller.js
index 586743e..d1b8900 100644
--- a/modules/web-console/frontend/app/components/list-editable/controller.js
+++ b/modules/web-console/frontend/app/components/list-editable/controller.js
@@ -102,10 +102,12 @@ export default class {
// ng-model-option, then it will always save. Be careful and pay extra attention to validation
// when doing so, it's an easy way to miss invalid values this way.
- // Dont close if form is invalid and allowInvalid is turned off (which is default value)
- if (!form.$valid && !this.ngModel.$options.getOption('allowInvalid')) return;
+ // Don't close if form is invalid and allowInvalid is turned off (which is default value)
+ if (!form.$valid && !this.ngModel.$options.getOption('allowInvalid'))
+ return;
delete this._cache[idx];
+
this.save(data, idx);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
index cd8d3e4..881e543 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
@@ -44,8 +44,6 @@ export default class ClusterEditFormController {
$onInit() {
this.available = this.IgniteVersion.available.bind(this.IgniteVersion);
- let __original_value;
-
const rebuildDropdowns = () => {
this.eventStorage = [
{value: 'Memory', label: 'Memory'},
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js
index 60cb95f..b7d4ebe 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js
@@ -171,22 +171,30 @@ export default class ModelEditFormController {
if ('caches' in changes)
this.cachesMenu = (changes.caches.currentValue || []).map((c) => ({label: c.name, value: c._id}));
}
+
/**
* @param {ig.config.model.DomainModel} model
*/
onQueryFieldsChange(model) {
this.$scope.backupItem = this.Models.removeInvalidFields(model);
}
+
getValuesToCompare() {
return [this.model, this.$scope.backupItem].map(this.Models.normalize);
}
+
save(download) {
if (this.$scope.ui.inputForm.$invalid)
return this.IgniteFormUtils.triggerValidation(this.$scope.ui.inputForm, this.$scope);
- if (!this.validate(this.$scope.backupItem)) return;
+
+ if (!this.validate(this.$scope.backupItem))
+ return;
+
this.onSave({$event: {model: cloneDeep(this.$scope.backupItem), download}});
}
+
reset = (forReal) => forReal ? this.$scope.backupItem = cloneDeep(this.model) : void 0;
+
confirmAndReset() {
return this.Confirm.confirm('Are you sure you want to undo all changes for current model?').then(() => true)
.then(this.reset)
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure-basic/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure-basic/controller.js b/modules/web-console/frontend/app/components/page-configure-basic/controller.js
index 255bbe5..105765a 100644
--- a/modules/web-console/frontend/app/components/page-configure-basic/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-basic/controller.js
@@ -81,8 +81,11 @@ export default class PageConfigureBasicController {
}
_uiCanExit($transition$) {
- if ($transition$.options().custom.justIDUpdate) return true;
+ if ($transition$.options().custom.justIDUpdate)
+ return true;
+
$transition$.onSuccess({}, () => this.reset());
+
return Observable.forkJoin(
this.ConfigureState.state$.pluck('edit', 'changes').take(1),
this.clusterID$.switchMap((id) => this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterShortCaches(id))).take(1),
@@ -178,7 +181,9 @@ export default class PageConfigureBasicController {
}
save(download = false) {
- if (this.form.$invalid) return this.IgniteFormUtils.triggerValidation(this.form, this.$scope);
+ if (this.form.$invalid)
+ return this.IgniteFormUtils.triggerValidation(this.form, this.$scope);
+
this.ConfigureState.dispatchAction((download ? basicSaveAndDownload : basicSave)(cloneDeep(this.clonedCluster)));
}
@@ -189,7 +194,7 @@ export default class PageConfigureBasicController {
confirmAndReset() {
return this.Confirm.confirm('Are you sure you want to undo all changes for current cluster?')
- .then(() => this.reset())
- .catch(() => {});
+ .then(() => this.reset())
+ .catch(() => {});
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure-overview/components/pco-grid-column-categories/directive.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure-overview/components/pco-grid-column-categories/directive.js b/modules/web-console/frontend/app/components/page-configure-overview/components/pco-grid-column-categories/directive.js
index 27d00dc..ff03e7f 100644
--- a/modules/web-console/frontend/app/components/page-configure-overview/components/pco-grid-column-categories/directive.js
+++ b/modules/web-console/frontend/app/components/page-configure-overview/components/pco-grid-column-categories/directive.js
@@ -38,7 +38,9 @@ export default function directive(uiGridConstants) {
require: '^uiGrid',
link: {
pre(scope, el, attr, grid) {
- if (!grid.grid.options.enableColumnCategories) return;
+ if (!grid.grid.options.enableColumnCategories)
+ return;
+
grid.grid.api.core.registerColumnsProcessor((cp) => {
const oldCategories = grid.grid.options.categories;
const newCategories = uniqBy(cp.filter(notSelectionColumn).map(({colDef: cd}) => {
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js b/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js
index 0225c5f..c962161 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js
@@ -19,6 +19,7 @@ import {default as ConfigChangesGuard} from '../services/ConfigChangesGuard';
class FormUICanExitGuardController {
static $inject = ['$element', ConfigChangesGuard.name];
+
/**
* @param {JQLite} $element
* @param {ConfigChangesGuard} ConfigChangesGuard
@@ -27,23 +28,30 @@ class FormUICanExitGuardController {
this.$element = $element;
this.ConfigChangesGuard = ConfigChangesGuard;
}
+
$onDestroy() {
this.$element = null;
}
+
$onInit() {
const data = this.$element.data();
const controller = Object.keys(data)
.map((key) => data[key])
.find(this._itQuacks);
- if (!controller) return;
+ if (!controller)
+ return;
controller.uiCanExit = ($transition$) => {
- if ($transition$.options().custom.justIDUpdate) return true;
+ if ($transition$.options().custom.justIDUpdate)
+ return true;
+
$transition$.onSuccess({}, controller.reset);
+
return this.ConfigChangesGuard.guard(...controller.getValuesToCompare());
};
}
+
_itQuacks(controller) {
return controller.reset instanceof Function &&
controller.getValuesToCompare instanceof Function &&
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
index 813c998..1d4ba7c 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
@@ -123,11 +123,12 @@ export class ModalImportModels {
this.ActivitiesData = ActivitiesData;
Object.assign(this, {Confirm, Focus, Messages, Loading, FormUtils, LegacyUtils});
}
+
loadData() {
return Observable.of(this.clusterID)
- .switchMap((id = 'new') => {
- return this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterToEdit(id, defaultNames.importedCluster));
- })
+ .switchMap((id = 'new') => {
+ return this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterToEdit(id, defaultNames.importedCluster));
+ })
.switchMap((cluster) => {
return (!(cluster.caches || []).length && !(cluster.models || []).length)
? Observable.of({
@@ -154,7 +155,9 @@ export class ModalImportModels {
.take(1);
}
saveBatch(batch) {
- if (!batch.length) return;
+ if (!batch.length)
+ return;
+
this.Loading.start('importDomainFromDb');
this.ConfigureState.dispatchAction({
type: 'ADVANCED_SAVE_COMPLETE_CONFIGURATION',
@@ -212,8 +215,12 @@ export class ModalImportModels {
return this.visibleTables = rows.map((r) => r.entity);
}
onCacheSelect(cacheID) {
- if (cacheID < 0) return;
- if (this.loadedCaches[cacheID]) return;
+ if (cacheID < 0)
+ return;
+
+ if (this.loadedCaches[cacheID])
+ return;
+
return this.onCacheSelectSubcription = Observable.merge(
Observable.timer(0, 1).take(1)
.do(() => this.ConfigureState.dispatchAction({type: 'LOAD_CACHE', cacheID})),
@@ -290,10 +297,10 @@ export class ModalImportModels {
}).subscribe();
// New
-
this.loadedCaches = {
...CACHE_TEMPLATES.reduce((a, c) => ({...a, [c.value]: c.cache}), {})
};
+
this.actions = [
{value: 'connect', label: this.$root.IgniteDemoMode ? 'Description' : 'Connection'},
{value: 'schemas', label: 'Schemas'},
@@ -302,7 +309,6 @@ export class ModalImportModels {
];
// Legacy
-
$scope.ui.invalidKeyFieldsTooltip = 'Found key types without configured key fields<br/>' +
'It may be a result of import tables from database without primary keys<br/>' +
'Key field for such key types should be configured manually';
@@ -591,7 +597,6 @@ export class ModalImportModels {
if ($scope._curDbTable.actionWatch) {
$scope._curDbTable.actionWatch();
-
$scope._curDbTable.actionWatch = null;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js
index 072a497..7fde5ba 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js
@@ -22,23 +22,33 @@ const IMPORT_DM_NEW_CACHE = 1;
export class TablesActionCell {
static $inject = ['$element'];
+
constructor($element) {
Object.assign(this, {$element});
}
+
onClick(e) {
e.stopPropagation();
}
+
$postLink() {
this.$element.on('click', this.onClick);
}
+
$onDestroy() {
this.$element.off('click', this.onClick);
this.$element = null;
}
+
tableActionView(table) {
- if (!this.caches) return;
+ if (!this.caches)
+ return;
+
const cache = this.caches.find((c) => c.value === table.cacheOrTemplate);
- if (!cache) return;
+
+ if (!cache)
+ return;
+
const cacheName = cache.label;
if (table.action === IMPORT_DM_NEW_CACHE)
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/controller.js b/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/controller.js
index e67ed25..1480fc1 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/controller.js
@@ -47,9 +47,15 @@ export default class ModalPreviewProjectController {
showPreview(node) {
this.fileText = '';
- if (!node) return;
+
+ if (!node)
+ return;
+
this.fileExt = node.file.name.split('.').reverse()[0].toLowerCase();
- if (node.file.dir) return;
+
+ if (node.file.dir)
+ return;
+
node.file.async('string').then((text) => {
this.fileText = text;
this.$scope.$applyAsync();
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/controller.js b/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/controller.js
index 3253fe4..0d751e8 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/controller.js
@@ -115,12 +115,16 @@ export default class PCFormFieldSizeController {
}
_defaultLabel() {
- if (!this.sizesMenu) return;
+ if (!this.sizesMenu)
+ return;
+
return this.sizesMenu[1].label;
}
chooseSizeScale(label = this._defaultLabel()) {
- if (!label) return;
+ if (!label)
+ return;
+
return this.sizesMenu.find((option) => option.label.toLowerCase() === label.toLowerCase());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/controller.js b/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/controller.js
index f4d1f47..c97e50f 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/controller.js
@@ -44,23 +44,28 @@ export default class ItemsTableController {
},
onRegisterApi: (api) => {
this.gridAPI = api;
+
api.selection.on.rowSelectionChanged(this.$scope, (row, e) => {
this.onRowsSelectionChange([row], e);
});
+
api.selection.on.rowSelectionChangedBatch(this.$scope, (rows, e) => {
this.onRowsSelectionChange(rows, e);
});
+
api.core.on.rowsVisibleChanged(this.$scope, () => {
const visibleRows = api.core.getVisibleRows();
if (this.onVisibleRowsChange) this.onVisibleRowsChange({$event: visibleRows});
this.adjustHeight(api, visibleRows.length);
this.showFilterNotification = this.grid.data.length && visibleRows.length === 0;
});
+
if (this.onFilterChanged) {
api.core.on.filterChanged(this.$scope, () => {
this.onFilterChanged();
});
}
+
this.$timeout(() => {
if (this.selectedRowId) this.applyIncomingSelection(this.selectedRowId);
});
@@ -73,10 +78,16 @@ export default class ItemsTableController {
oneWaySelection = false;
onRowsSelectionChange = debounce((rows, e = {}) => {
- if (e.ignore) return;
+ if (e.ignore)
+ return;
+
const selected = this.gridAPI.selection.legacyGetSelectedRows();
- if (this.oneWaySelection) rows.forEach((r) => r.isSelected = false);
- if (this.onSelectionChange) this.onSelectionChange({$event: selected});
+
+ if (this.oneWaySelection)
+ rows.forEach((r) => r.isSelected = false);
+
+ if (this.onSelectionChange)
+ this.onSelectionChange({$event: selected});
});
makeActionsMenu(incomingActionsMenu = []) {
@@ -85,17 +96,22 @@ export default class ItemsTableController {
$onChanges(changes) {
const hasChanged = (binding) => binding in changes && changes[binding].currentValue !== changes[binding].previousValue;
+
if (hasChanged('items') && this.grid) {
this.grid.data = changes.items.currentValue;
this.gridAPI.grid.modifyRows(this.grid.data);
this.adjustHeight(this.gridAPI, this.grid.data.length);
+
// Without property existence check non-set selectedRowId binding might cause
// unwanted behavior, like unchecking rows during any items change, even if
// nothing really changed.
- if ('selectedRowId' in this) this.applyIncomingSelection(this.selectedRowId);
+ if ('selectedRowId' in this)
+ this.applyIncomingSelection(this.selectedRowId);
}
+
if (hasChanged('selectedRowId') && this.grid && this.grid.data)
this.applyIncomingSelection(changes.selectedRowId.currentValue);
+
if ('incomingActionsMenu' in changes)
this.actionsMenu = this.makeActionsMenu(changes.incomingActionsMenu.currentValue);
}
@@ -103,9 +119,11 @@ export default class ItemsTableController {
applyIncomingSelection(selected = []) {
this.gridAPI.selection.clearSelectedRows({ignore: true});
const rows = this.grid.data.filter((r) => selected.includes(r[this.rowIdentityKey]));
+
rows.forEach((r) => {
this.gridAPI.selection.selectRow(r, {ignore: true});
});
+
if (rows.length === 1) {
this.$timeout(() => {
this.gridAPI.grid.scrollToIfNecessary(this.gridAPI.grid.getRow(rows[0]), null);
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/directive.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/directive.js b/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/directive.js
index ccbdb8d..3d4617c 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/directive.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/directive.js
@@ -23,7 +23,9 @@ export default function pcUiGridFilters(uiGridConstants) {
require: 'uiGrid',
link: {
pre(scope, el, attr, grid) {
- if (!grid.grid.options.enableFiltering) return;
+ if (!grid.grid.options.enableFiltering)
+ return;
+
grid.grid.options.columnDefs.filter((cd) => cd.multiselectFilterOptions).forEach((cd) => {
cd.headerCellTemplate = template;
cd.filter = {
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/index.js b/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/index.js
index 4ec96e2..a34a78a 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/index.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/index.js
@@ -27,9 +27,12 @@ export default angular
instance.$referenceElement = el;
instance.destroy = flow(instance.destroy, () => instance.$referenceElement = null);
instance.$applyPlacement = flow(instance.$applyPlacement, () => {
- if (!instance.$element) return;
+ if (!instance.$element)
+ return;
+
const refWidth = instance.$referenceElement[0].getBoundingClientRect().width;
const elWidth = instance.$element[0].getBoundingClientRect().width;
+
if (refWidth > elWidth) {
instance.$element.css({
width: refWidth,
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/components/pcIsInCollection.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pcIsInCollection.js b/modules/web-console/frontend/app/components/page-configure/components/pcIsInCollection.js
index d2a9ae8..50ea4a4 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pcIsInCollection.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/pcIsInCollection.js
@@ -18,7 +18,9 @@
class Controller {
$onInit() {
this.ngModel.$validators.isInCollection = (item) => {
- if (!item || !this.items) return true;
+ if (!item || !this.items)
+ return true;
+
return this.items.includes(item);
};
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js b/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js
index e4c7314..1702c35 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js
@@ -56,7 +56,9 @@ export default angular.module('ignite-console.page-configure.validation', [])
$onInit() {
this.ngModel.$validators.notInCollection = (item) => {
- if (!this.items) return true;
+ if (!this.items)
+ return true;
+
return !this.items.includes(item);
};
}
@@ -87,7 +89,9 @@ export default angular.module('ignite-console.page-configure.validation', [])
$onInit() {
this.ngModel.$validators.inCollection = (item) => {
- if (!this.items) return false;
+ if (!this.items)
+ return false;
+
const items = this.pluck ? this.items.map((i) => i[this.pluck]) : this.items;
return Array.isArray(item)
? item.every((i) => items.includes(i))
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/index.js b/modules/web-console/frontend/app/components/page-configure/index.js
index 0b254a0..9beea7a 100644
--- a/modules/web-console/frontend/app/components/page-configure/index.js
+++ b/modules/web-console/frontend/app/components/page-configure/index.js
@@ -61,6 +61,7 @@ Observable.prototype.debug = function(l) {
import {
editReducer2,
+ shortObjectsReducer,
reducer,
editReducer,
loadingReducer,
@@ -124,16 +125,18 @@ export default angular
});
ConfigureState.actions$
- .filter((e) => e.type !== 'DISPATCH')
- .withLatestFrom(ConfigureState.state$.skip(1))
- .subscribe(([action, state]) => devTools.send(action, state));
+ .filter((e) => e.type !== 'DISPATCH')
+ .withLatestFrom(ConfigureState.state$.skip(1))
+ .subscribe(([action, state]) => devTools.send(action, state));
ConfigureState.addReducer(reduxDevtoolsReducer);
}
+
ConfigureState.addReducer(refsReducer({
models: {at: 'domains', store: 'caches'},
caches: {at: 'caches', store: 'models'}
}));
+
ConfigureState.addReducer((state, action) => Object.assign({}, state, {
clusterConfiguration: editReducer(state.clusterConfiguration, action),
configurationLoading: loadingReducer(state.configurationLoading, action),
@@ -148,14 +151,19 @@ export default angular
shortIgfss: mapCacheReducerFactory(shortIGFSsActionTypes)(state.shortIgfss, action),
edit: editReducer2(state.edit, action)
}));
+
+ ConfigureState.addReducer(shortObjectsReducer);
+
ConfigureState.addReducer((state, action) => {
switch (action.type) {
case 'APPLY_ACTIONS_UNDO':
return action.state;
+
default:
return state;
}
});
+
const la = ConfigureState.actions$.scan((acc, action) => [...acc, action], []);
ConfigureState.actions$
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/reducer.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/reducer.js b/modules/web-console/frontend/app/components/page-configure/reducer.js
index f959b0a..0f24bfa 100644
--- a/modules/web-console/frontend/app/components/page-configure/reducer.js
+++ b/modules/web-console/frontend/app/components/page-configure/reducer.js
@@ -16,6 +16,7 @@
*/
import difference from 'lodash/difference';
+import capitalize from 'lodash/capitalize';
export const LOAD_LIST = Symbol('LOAD_LIST');
export const ADD_CLUSTER = Symbol('ADD_CLUSTER');
@@ -33,6 +34,7 @@ import {
} from './store/actionTypes';
const defaults = {clusters: new Map(), caches: new Map(), spaces: new Map()};
+
const mapByID = (items) => {
return Array.isArray(items) ? new Map(items.map((item) => [item._id, item])) : new Map(items);
};
@@ -48,21 +50,25 @@ export const reducer = (state = defaults, action) => {
plugins: mapByID(action.list.plugins)
};
}
+
case ADD_CLUSTER: {
return Object.assign({}, state, {
clusters: new Map([...state.clusters.entries(), [action.cluster._id, action.cluster]])
});
}
+
case ADD_CLUSTERS: {
return Object.assign({}, state, {
clusters: new Map([...state.clusters.entries(), ...action.clusters.map((c) => [c._id, c])])
});
}
+
case REMOVE_CLUSTERS: {
return Object.assign({}, state, {
clusters: new Map([...state.clusters.entries()].filter(([id, value]) => !action.clusterIDs.includes(id)))
});
}
+
case UPDATE_CLUSTER: {
const id = action._id || action.cluster._id;
return Object.assign({}, state, {
@@ -74,31 +80,38 @@ export const reducer = (state = defaults, action) => {
}))
});
}
+
case UPSERT_CLUSTERS: {
return action.clusters.reduce((state, cluster) => reducer(state, {
type: state.clusters.has(cluster._id) ? UPDATE_CLUSTER : ADD_CLUSTER,
cluster
}), state);
}
+
case ADD_CACHE: {
return Object.assign({}, state, {
caches: new Map([...state.caches.entries(), [action.cache._id, action.cache]])
});
}
+
case UPDATE_CACHE: {
const id = action.cache._id;
+
return Object.assign({}, state, {
caches: new Map(state.caches).set(id, Object.assign({}, state.caches.get(id), action.cache))
});
}
+
case UPSERT_CACHES: {
return action.caches.reduce((state, cache) => reducer(state, {
type: state.caches.has(cache._id) ? UPDATE_CACHE : ADD_CACHE,
cache
}), state);
}
+
case REMOVE_CACHE:
return state;
+
default:
return state;
}
@@ -119,34 +132,40 @@ export const editReducer = (state = {originalCluster: null}, action) => {
...state,
originalCluster: action.cluster
};
+
case RECEIVE_CACHE_EDIT: {
return {
...state,
originalCache: action.cache
};
}
+
case RECEIVE_IGFSS_EDIT:
return {
...state,
originalIGFSs: action.igfss
};
+
case RECEIVE_IGFS_EDIT: {
return {
...state,
originalIGFS: action.igfs
};
}
+
case RECEIVE_MODELS_EDIT:
return {
...state,
originalModels: action.models
};
+
case RECEIVE_MODEL_EDIT: {
return {
...state,
originalModel: action.model
};
}
+
default:
return state;
}
@@ -160,8 +179,10 @@ export const loadingReducer = (state = loadingDefaults, action) => {
switch (action.type) {
case SHOW_CONFIG_LOADING:
return {...state, isLoading: true, loadingText: action.loadingText};
+
case HIDE_CONFIG_LOADING:
return {...state, isLoading: false};
+
default:
return state;
}
@@ -171,12 +192,16 @@ export const setStoreReducerFactory = (actionTypes) => (state = new Set(), actio
switch (action.type) {
case actionTypes.SET:
return new Set(action.items.map((i) => i._id));
+
case actionTypes.RESET:
return new Set();
+
case actionTypes.UPSERT:
return action.items.reduce((acc, item) => {acc.add(item._id); return acc;}, new Set(state));
+
case actionTypes.REMOVE:
return action.items.reduce((acc, item) => {acc.delete(item); return acc;}, new Set(state));
+
default:
return state;
}
@@ -186,14 +211,22 @@ export const mapStoreReducerFactory = (actionTypes) => (state = new Map(), actio
switch (action.type) {
case actionTypes.SET:
return new Map(action.items.map((i) => [i._id, i]));
+
case actionTypes.RESET:
return new Map();
+
case actionTypes.UPSERT:
- if (!action.items.length) return state;
+ if (!action.items.length)
+ return state;
+
return action.items.reduce((acc, item) => {acc.set(item._id, item); return acc;}, new Map(state));
+
case actionTypes.REMOVE:
- if (!action.ids.length) return state;
+ if (!action.ids.length)
+ return state;
+
return action.ids.reduce((acc, id) => {acc.delete(id); return acc;}, new Map(state));
+
default:
return state;
}
@@ -201,6 +234,7 @@ export const mapStoreReducerFactory = (actionTypes) => (state = new Map(), actio
export const mapCacheReducerFactory = (actionTypes) => {
const mapStoreReducer = mapStoreReducerFactory(actionTypes);
+
return (state = {value: mapStoreReducer(), pristine: true}, action) => {
switch (action.type) {
case actionTypes.SET:
@@ -210,11 +244,13 @@ export const mapCacheReducerFactory = (actionTypes) => {
value: mapStoreReducer(state.value, action),
pristine: false
};
+
case actionTypes.RESET:
return {
value: mapStoreReducer(state.value, action),
pristine: true
};
+
default:
return state;
}
@@ -248,26 +284,31 @@ export const shortIGFSsActionTypes = mapStoreActionTypesFactory('SHORT_IGFSS');
export const itemsEditReducerFactory = (actionTypes) => {
const setStoreReducer = setStoreReducerFactory(actionTypes);
const mapStoreReducer = mapStoreReducerFactory(actionTypes);
+
return (state = {ids: setStoreReducer(), changedItems: mapStoreReducer()}, action) => {
switch (action.type) {
case actionTypes.SET:
return action.state;
+
case actionTypes.LOAD:
return {
...state,
ids: setStoreReducer(state.ids, {...action, type: actionTypes.UPSERT})
};
+
case actionTypes.RESET:
case actionTypes.UPSERT:
return {
ids: setStoreReducer(state.ids, action),
changedItems: mapStoreReducer(state.changedItems, action)
};
+
case actionTypes.REMOVE:
return {
ids: setStoreReducer(state.ids, {type: action.type, items: action.ids}),
changedItems: mapStoreReducer(state.changedItems, action)
};
+
default:
return state;
}
@@ -278,6 +319,7 @@ export const editReducer2 = (state = editReducer2.getDefaults(), action) => {
switch (action.type) {
case 'SET_EDIT':
return action.state;
+
case 'EDIT_CLUSTER': {
return {
...state,
@@ -293,6 +335,7 @@ export const editReducer2 = (state = editReducer2.getDefaults(), action) => {
}
};
}
+
case 'RESET_EDIT_CHANGES': {
return {
...state,
@@ -308,6 +351,7 @@ export const editReducer2 = (state = editReducer2.getDefaults(), action) => {
}
};
}
+
case 'UPSERT_CLUSTER': {
return {
...state,
@@ -317,6 +361,7 @@ export const editReducer2 = (state = editReducer2.getDefaults(), action) => {
}
};
}
+
case 'UPSERT_CLUSTER_ITEM': {
const {itemType, item} = action;
return {
@@ -330,8 +375,10 @@ export const editReducer2 = (state = editReducer2.getDefaults(), action) => {
}
};
}
+
case REMOVE_CLUSTER_ITEMS_CONFIRMED: {
const {itemType, itemIDs} = action;
+
return {
...state,
changes: {
@@ -343,9 +390,11 @@ export const editReducer2 = (state = editReducer2.getDefaults(), action) => {
}
};
}
+
default: return state;
}
};
+
editReducer2.getDefaults = () => ({
changes: ['caches', 'models', 'igfss'].reduce((a, t) => ({...a, [t]: {ids: [], changedItems: []}}), {cluster: null})
});
@@ -356,15 +405,19 @@ export const refsReducer = (refs) => (state, action) => {
const newCluster = action.changedItems.cluster;
const oldCluster = state.clusters.get(newCluster._id) || {};
const val = Object.keys(refs).reduce((state, ref) => {
- if (!state || !state[refs[ref].store].size) return state;
+ if (!state || !state[refs[ref].store].size)
+ return state;
const addedSources = new Set(difference(newCluster[ref], oldCluster[ref] || []));
const removedSources = new Set(difference(oldCluster[ref] || [], newCluster[ref]));
const changedSources = new Map(action.changedItems[ref].map((m) => [m._id, m]));
const targets = new Map();
+
const maybeTarget = (id) => {
- if (!targets.has(id)) targets.set(id, {[refs[ref].at]: {add: new Set(), remove: new Set()}});
+ if (!targets.has(id))
+ targets.set(id, {[refs[ref].at]: {add: new Set(), remove: new Set()}});
+
return targets.get(id);
};
@@ -373,11 +426,13 @@ export const refsReducer = (refs) => (state, action) => {
.filter((sourceID) => removedSources.has(sourceID))
.forEach((sourceID) => maybeTarget(target._id)[refs[ref].at].remove.add(sourceID));
});
+
[...addedSources.values()].forEach((sourceID) => {
(changedSources.get(sourceID)[refs[ref].store] || []).forEach((targetID) => {
maybeTarget(targetID)[refs[ref].at].add.add(sourceID);
});
});
+
action.changedItems[ref].filter((s) => !addedSources.has(s._id)).forEach((source) => {
const newSource = source;
const oldSource = state[ref].get(source._id);
@@ -412,8 +467,35 @@ export const refsReducer = (refs) => (state, action) => {
}
: state;
}, state);
+
return val;
}
+
+ default:
+ return state;
+ }
+};
+
+export const shortObjectsReducer = (state, action) => {
+ switch (action.type) {
+ case REMOVE_CLUSTER_ITEMS_CONFIRMED: {
+ const {itemType, itemIDs} = action;
+
+ const target = 'short' + capitalize(itemType);
+
+ const oldItems = state[target];
+
+ const newItems = {
+ value: itemIDs.reduce((acc, id) => {acc.delete(id); return acc;}, oldItems.value),
+ pristine: oldItems.pristine
+ };
+
+ return {
+ ...state,
+ [target]: newItems
+ };
+ }
+
default:
return state;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js b/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js
index ea7f527..27e227d 100644
--- a/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js
+++ b/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js
@@ -32,15 +32,19 @@ export default class ConfigureState {
return this._combinedReducer(state, action);
} catch (e) {
console.error(e);
+
return state;
}
};
+
this.actions$.scan(reducer, {}).do((v) => this.state$.next(v)).subscribe();
}
addReducer(combineFn) {
const old = this._combinedReducer;
+
this._combinedReducer = (state, action) => combineFn(old(state, action), action);
+
return this;
}
@@ -49,6 +53,7 @@ export default class ConfigureState {
return action((a) => this.actions$.next(a), () => this.state$.getValue());
this.actions$.next(action);
+
return action;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/states.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/states.js b/modules/web-console/frontend/app/components/page-configure/states.js
index e76b18e..ed43f48 100644
--- a/modules/web-console/frontend/app/components/page-configure/states.js
+++ b/modules/web-console/frontend/app/components/page-configure/states.js
@@ -25,13 +25,15 @@ import {Observable} from 'rxjs/Observable';
const idRegex = `new|[a-z0-9]+`;
const shortCachesResolve = ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
- if ($transition$.params().clusterID === 'new') return Promise.resolve();
+ if ($transition$.params().clusterID === 'new')
+ return Promise.resolve();
+
return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
- .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
- .switchMap((cluster) => {
- return etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id});
- })
- .toPromise();
+ .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+ .switchMap((cluster) => {
+ return etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id});
+ })
+ .toPromise();
}];
function registerStates($stateProvider) {
@@ -134,17 +136,19 @@ function registerStates($stateProvider) {
component: pageConfigureAdvancedCachesComponent.name,
resolve: {
_shortCachesAndModels: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
- if ($transition$.params().clusterID === 'new') return Promise.resolve();
+ if ($transition$.params().clusterID === 'new')
+ return Promise.resolve();
+
return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
- .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
- .map((cluster) => {
- return Promise.all([
- etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
- etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id}),
- etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
- ]);
- })
- .toPromise();
+ .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+ .map((cluster) => {
+ return Promise.all([
+ etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
+ etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id}),
+ etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
+ ]);
+ })
+ .toPromise();
}]
},
resolvePolicy: {
@@ -160,7 +164,9 @@ function registerStates($stateProvider) {
resolve: {
_cache: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
const {clusterID, cacheID} = $transition$.params();
- if (cacheID === 'new') return Promise.resolve();
+ if (cacheID === 'new')
+ return Promise.resolve();
+
return etp('LOAD_CACHE', {cacheID});
}]
},
@@ -180,16 +186,18 @@ function registerStates($stateProvider) {
permission: 'configuration',
resolve: {
_shortCachesAndModels: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
- if ($transition$.params().clusterID === 'new') return Promise.resolve();
+ if ($transition$.params().clusterID === 'new')
+ return Promise.resolve();
+
return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
- .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
- .map((cluster) => {
- return Promise.all([
- etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
- etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id})
- ]);
- })
- .toPromise();
+ .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+ .map((cluster) => {
+ return Promise.all([
+ etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
+ etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id})
+ ]);
+ })
+ .toPromise();
}]
},
resolvePolicy: {
@@ -204,7 +212,10 @@ function registerStates($stateProvider) {
resolve: {
_cache: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
const {clusterID, modelID} = $transition$.params();
- if (modelID === 'new') return Promise.resolve();
+
+ if (modelID === 'new')
+ return Promise.resolve();
+
return etp('LOAD_MODEL', {modelID});
}]
},
@@ -222,15 +233,17 @@ function registerStates($stateProvider) {
permission: 'configuration',
resolve: {
_shortIGFSs: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
- if ($transition$.params().clusterID === 'new') return Promise.resolve();
+ if ($transition$.params().clusterID === 'new')
+ return Promise.resolve();
+
return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
- .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
- .map((cluster) => {
- return Promise.all([
- etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
- ]);
- })
- .toPromise();
+ .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+ .map((cluster) => {
+ return Promise.all([
+ etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
+ ]);
+ })
+ .toPromise();
}]
},
resolvePolicy: {
@@ -246,7 +259,10 @@ function registerStates($stateProvider) {
resolve: {
_igfs: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
const {clusterID, igfsID} = $transition$.params();
- if (igfsID === 'new') return Promise.resolve();
+
+ if (igfsID === 'new')
+ return Promise.resolve();
+
return etp('LOAD_IGFS', {igfsID});
}]
},
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/store/effects.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/store/effects.js b/modules/web-console/frontend/app/components/page-configure/store/effects.js
index 42c4023..072701a 100644
--- a/modules/web-console/frontend/app/components/page-configure/store/effects.js
+++ b/modules/web-console/frontend/app/components/page-configure/store/effects.js
@@ -26,41 +26,39 @@ import {fromPromise} from 'rxjs/observable/fromPromise';
import uniq from 'lodash/uniq';
import {
- clustersActionTypes,
cachesActionTypes,
- shortClustersActionTypes,
+ clustersActionTypes,
+ igfssActionTypes,
+ modelsActionTypes,
shortCachesActionTypes,
- shortModelsActionTypes,
+ shortClustersActionTypes,
shortIGFSsActionTypes,
- modelsActionTypes,
- igfssActionTypes
+ shortModelsActionTypes
} from '../reducer';
import {
- REMOVE_CLUSTER_ITEMS,
- REMOVE_CLUSTER_ITEMS_CONFIRMED,
- CONFIRM_CLUSTERS_REMOVAL,
- CONFIRM_CLUSTERS_REMOVAL_OK,
- COMPLETE_CONFIGURATION,
- ADVANCED_SAVE_COMPLETE_CONFIGURATION,
- ADVANCED_SAVE_CLUSTER,
ADVANCED_SAVE_CACHE,
+ ADVANCED_SAVE_CLUSTER,
+ ADVANCED_SAVE_COMPLETE_CONFIGURATION,
ADVANCED_SAVE_IGFS,
ADVANCED_SAVE_MODEL,
BASIC_SAVE,
BASIC_SAVE_AND_DOWNLOAD,
- BASIC_SAVE_OK
+ BASIC_SAVE_OK,
+ COMPLETE_CONFIGURATION,
+ CONFIRM_CLUSTERS_REMOVAL,
+ CONFIRM_CLUSTERS_REMOVAL_OK,
+ REMOVE_CLUSTER_ITEMS,
+ REMOVE_CLUSTER_ITEMS_CONFIRMED
} from './actionTypes';
import {
- removeClusterItemsConfirmed,
advancedSaveCompleteConfiguration,
- confirmClustersRemoval,
- confirmClustersRemovalOK,
- completeConfiguration,
- basicSave,
+ basicSaveErr,
basicSaveOK,
- basicSaveErr
+ completeConfiguration,
+ confirmClustersRemovalOK,
+ removeClusterItemsConfirmed
} from './actionCreators';
import ConfigureState from 'app/components/page-configure/services/ConfigureState';
@@ -267,12 +265,14 @@ export default class ConfigEffects {
.exhaustMap((a) => {
return this.ConfigureState.state$.let(this.ConfigSelectors.selectCache(a.cacheID)).take(1)
.switchMap((cache) => {
- if (cache) return of({type: `${a.type}_OK`, cache});
+ if (cache)
+ return of({type: `${a.type}_OK`, cache});
+
return fromPromise(this.Caches.getCache(a.cacheID))
- .switchMap(({data}) => of(
- {type: 'CACHE', cache: data},
- {type: `${a.type}_OK`, cache: data}
- ));
+ .switchMap(({data}) => of(
+ {type: 'CACHE', cache: data},
+ {type: `${a.type}_OK`, cache: data}
+ ));
})
.catch((error) => of({
type: `${a.type}_ERR`,
@@ -289,7 +289,9 @@ export default class ConfigEffects {
this.loadShortCachesEffect$ = ConfigureState.actions$
.let(ofType('LOAD_SHORT_CACHES'))
.exhaustMap((a) => {
- if (!(a.ids || []).length) return of({type: `${a.type}_OK`});
+ if (!(a.ids || []).length)
+ return of({type: `${a.type}_OK`});
+
return this.ConfigureState.state$.let(this.ConfigSelectors.selectShortCaches()).take(1)
.switchMap((items) => {
if (!items.pristine && a.ids && a.ids.every((_id) => items.value.has(_id)))
@@ -315,12 +317,14 @@ export default class ConfigEffects {
.exhaustMap((a) => {
return this.ConfigureState.state$.let(this.ConfigSelectors.selectIGFS(a.igfsID)).take(1)
.switchMap((igfs) => {
- if (igfs) return of({type: `${a.type}_OK`, igfs});
+ if (igfs)
+ return of({type: `${a.type}_OK`, igfs});
+
return fromPromise(this.IGFSs.getIGFS(a.igfsID))
- .switchMap(({data}) => of(
- {type: 'IGFS', igfs: data},
- {type: `${a.type}_OK`, igfs: data}
- ));
+ .switchMap(({data}) => of(
+ {type: 'IGFS', igfs: data},
+ {type: `${a.type}_OK`, igfs: data}
+ ));
})
.catch((error) => of({
type: `${a.type}_ERR`,
@@ -368,12 +372,14 @@ export default class ConfigEffects {
.exhaustMap((a) => {
return this.ConfigureState.state$.let(this.ConfigSelectors.selectModel(a.modelID)).take(1)
.switchMap((model) => {
- if (model) return of({type: `${a.type}_OK`, model});
+ if (model)
+ return of({type: `${a.type}_OK`, model});
+
return fromPromise(this.Models.getModel(a.modelID))
- .switchMap(({data}) => of(
- {type: 'MODEL', model: data},
- {type: `${a.type}_OK`, model: data}
- ));
+ .switchMap(({data}) => of(
+ {type: 'MODEL', model: data},
+ {type: `${a.type}_OK`, model: data}
+ ));
})
.catch((error) => of({
type: `${a.type}_ERR`,
@@ -454,39 +460,48 @@ export default class ConfigEffects {
const go = (state, params = {}) => this.$state.go(
state, {...params, clusterID: cluster._id}, {location: 'replace', custom: {justIDUpdate: true}}
);
+
switch (type) {
case 'models': {
const state = 'base.configuration.edit.advanced.models.model';
this.IgniteMessages.showInfo(`Model "${value.valueType}" saved`);
- if (
- this.$state.is(state) && this.$state.params.modelID !== value._id
- ) return go(state, {modelID: value._id});
+
+ if (this.$state.is(state) && this.$state.params.modelID !== value._id)
+ return go(state, {modelID: value._id});
+
break;
}
+
case 'caches': {
const state = 'base.configuration.edit.advanced.caches.cache';
this.IgniteMessages.showInfo(`Cache "${value.name}" saved`);
- if (
- this.$state.is(state) && this.$state.params.cacheID !== value._id
- ) return go(state, {cacheID: value._id});
+
+ if (this.$state.is(state) && this.$state.params.cacheID !== value._id)
+ return go(state, {cacheID: value._id});
+
break;
}
+
case 'igfss': {
const state = 'base.configuration.edit.advanced.igfs.igfs';
this.IgniteMessages.showInfo(`IGFS "${value.name}" saved`);
- if (
- this.$state.is(state) && this.$state.params.igfsID !== value._id
- ) return go(state, {igfsID: value._id});
+
+ if (this.$state.is(state) && this.$state.params.igfsID !== value._id)
+ return go(state, {igfsID: value._id});
+
break;
}
+
case 'cluster': {
const state = 'base.configuration.edit.advanced.cluster';
this.IgniteMessages.showInfo(`Cluster "${value.name}" saved`);
- if (
- this.$state.is(state) && this.$state.params.clusterID !== value._id
- ) return go(state);
+
+ if (this.$state.is(state) && this.$state.params.clusterID !== value._id)
+ return go(state);
+
break;
}
+
default: break;
}
})
@@ -497,7 +512,7 @@ export default class ConfigEffects {
.exhaustMap((a) => {
return a.confirm
// TODO: list items to remove in confirmation
- ? fromPromise(this.Confirm.confirm('Are you sure want to remove these items?'))
+ ? fromPromise(this.Confirm.confirm('Are you sure you want to remove these items?'))
.mapTo(a)
.catch(() => empty())
: of(a);
@@ -522,7 +537,7 @@ export default class ConfigEffects {
.switchMap((ids) => this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterNames(ids)).take(1))
.exhaustMap((names) => {
return fromPromise(this.Confirm.confirm(`
- <p>Are you sure want to remove these clusters?</p>
+ <p>Are you sure you want to remove these clusters?</p>
<ul>${names.map((name) => `<li>${name}</li>`).join('')}</ul>
`))
.map(confirmClustersRemovalOK)
@@ -656,6 +671,7 @@ export default class ConfigEffects {
const err = `${action.type}_ERR`;
setTimeout(() => this.ConfigureState.dispatchAction(action));
+
return this.ConfigureState.actions$
.filter((a) => a.type === ok || a.type === err)
.take(1)
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/store/selectors.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/store/selectors.js b/modules/web-console/frontend/app/components/page-configure/store/selectors.js
index 0f60214..4996ba9 100644
--- a/modules/web-console/frontend/app/components/page-configure/store/selectors.js
+++ b/modules/web-console/frontend/app/components/page-configure/store/selectors.js
@@ -30,23 +30,37 @@ import {default as IGFSs} from 'app/services/IGFSs';
import {default as Models} from 'app/services/Models';
const isDefined = (s) => s.filter((v) => v);
+
const selectItems = (path) => (s) => s.filter((s) => s).pluck(path).filter((v) => v);
+
const selectValues = (s) => s.map((v) => v && [...v.value.values()]);
+
export const selectMapItem = (mapPath, key) => (s) => s.pluck(mapPath).map((v) => v && v.get(key));
+
const selectMapItems = (mapPath, keys) => (s) => s.pluck(mapPath).map((v) => v && keys.map((key) => v.get(key)));
+
const selectItemToEdit = ({items, itemFactory, defaultName = '', itemID}) => (s) => s.switchMap((item) => {
- if (item) return of(Object.assign(itemFactory(), item));
- if (itemID === 'new') return items.take(1).map((items) => Object.assign(itemFactory(), {name: uniqueName(defaultName, items)}));
- if (!itemID) return of(null);
+ if (item)
+ return of(Object.assign(itemFactory(), item));
+
+ if (itemID === 'new')
+ return items.take(1).map((items) => Object.assign(itemFactory(), {name: uniqueName(defaultName, items)}));
+
+ if (!itemID)
+ return of(null);
+
return empty();
});
+
const currentShortItems = ({changesKey, shortKey}) => (state$) => {
return Observable.combineLatest(
state$.pluck('edit', 'changes', changesKey).let(isDefined).distinctUntilChanged(),
state$.pluck(shortKey, 'value').let(isDefined).distinctUntilChanged()
)
.map(([{ids = [], changedItems}, shortItems]) => {
- if (!ids.length || !shortItems) return [];
+ if (!ids.length || !shortItems)
+ return [];
+
return ids.map((id) => changedItems.find(({_id}) => _id === id) || shortItems.get(id));
})
.map((v) => v.filter((v) => v));
@@ -58,6 +72,7 @@ const selectNames = (itemIDs, nameAt = 'name') => (items) => items
export default class ConfigSelectors {
static $inject = ['Caches', 'Clusters', 'IGFSs', 'Models'];
+
/**
* @param {Caches} Caches
* @param {Clusters} Clusters
@@ -91,15 +106,25 @@ export default class ConfigSelectors {
.let(this.selectShortClusters())
.let(selectNames(clusterIDs));
}
+
selectCluster = (id) => selectMapItem('clusters', id);
+
selectShortClusters = () => selectItems('shortClusters');
+
selectCache = (id) => selectMapItem('caches', id);
+
selectIGFS = (id) => selectMapItem('igfss', id);
+
selectShortCaches = () => selectItems('shortCaches');
+
selectShortCachesValue = () => (state$) => state$.let(this.selectShortCaches()).let(selectValues);
+
selectShortIGFSs = () => selectItems('shortIgfss');
+
selectShortIGFSsValue = () => (state$) => state$.let(this.selectShortIGFSs()).let(selectValues);
+
selectShortModelsValue = () => (state$) => state$.let(this.selectShortModels()).let(selectValues);
+
selectCacheToEdit = (cacheID) => (state$) => state$
.let(this.selectCache(cacheID))
.distinctUntilChanged()
@@ -109,6 +134,7 @@ export default class ConfigSelectors {
defaultName: defaultNames.cache,
itemID: cacheID
}));
+
selectIGFSToEdit = (itemID) => (state$) => state$
.let(this.selectIGFS(itemID))
.distinctUntilChanged()
@@ -118,6 +144,7 @@ export default class ConfigSelectors {
defaultName: defaultNames.igfs,
itemID
}));
+
selectModelToEdit = (itemID) => (state$) => state$
.let(this.selectModel(itemID))
.distinctUntilChanged()
@@ -126,6 +153,7 @@ export default class ConfigSelectors {
itemFactory: () => this.Models.getBlankModel(),
itemID
}));
+
selectClusterToEdit = (clusterID, defaultName = defaultNames.cluster) => (state$) => state$
.let(this.selectCluster(clusterID))
.distinctUntilChanged()
@@ -135,23 +163,33 @@ export default class ConfigSelectors {
defaultName,
itemID: clusterID
}));
+
selectCurrentShortCaches = currentShortItems({changesKey: 'caches', shortKey: 'shortCaches'});
+
selectCurrentShortIGFSs = currentShortItems({changesKey: 'igfss', shortKey: 'shortIgfss'});
+
selectCurrentShortModels = currentShortItems({changesKey: 'models', shortKey: 'shortModels'});
+
selectClusterShortCaches = (clusterID) => (state$) => {
- if (clusterID === 'new') return of([]);
+ if (clusterID === 'new')
+ return of([]);
+
return combineLatest(
state$.let(this.selectCluster(clusterID)).pluck('caches'),
state$.let(this.selectShortCaches()).pluck('value'),
(ids, items) => ids.map((id) => items.get(id))
);
};
+
selectCompleteClusterConfiguration = ({clusterID, isDemo}) => (state$) => {
const hasValues = (array) => !array.some((v) => !v);
return state$.let(this.selectCluster(clusterID))
.exhaustMap((cluster) => {
- if (!cluster) return of({__isComplete: false});
+ if (!cluster)
+ return of({__isComplete: false});
+
const withSpace = (array) => array.map((c) => ({...c, space: cluster.space}));
+
return Observable.forkJoin(
state$.let(selectMapItems('caches', cluster.caches || [])).take(1),
state$.let(selectMapItems('models', cluster.models || [])).take(1),
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-configure/transitionHooks/errorState.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-configure/transitionHooks/errorState.js b/modules/web-console/frontend/app/components/page-configure/transitionHooks/errorState.js
index f36a931..fd9ca9b 100644
--- a/modules/web-console/frontend/app/components/page-configure/transitionHooks/errorState.js
+++ b/modules/web-console/frontend/app/components/page-configure/transitionHooks/errorState.js
@@ -44,9 +44,12 @@ const getResolvePromises = ($transition) => $transition.getResolveTokens()
*/
export const errorState = ($uiRouter) => {
$uiRouter.transitionService.onError(match, ($transition) => {
- if ($transition.error().type !== RejectType.ERROR) return;
+ if ($transition.error().type !== RejectType.ERROR)
+ return;
+
go($transition);
});
+
$uiRouter.transitionService.onStart(match, ($transition) => {
Promise.all(getResolvePromises($transition)).catch((e) => go($transition));
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/page-signin/run.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-signin/run.js b/modules/web-console/frontend/app/components/page-signin/run.js
index 7ac3bf8..457e488 100644
--- a/modules/web-console/frontend/app/components/page-signin/run.js
+++ b/modules/web-console/frontend/app/components/page-signin/run.js
@@ -27,7 +27,10 @@ export function registerState($uiRouter) {
unsaved: true,
redirectTo: (trans) => {
const skipStates = new Set(['signup', 'forgotPassword', 'landing']);
- if (skipStates.has(trans.from().name)) return;
+
+ if (skipStates.has(trans.from().name))
+ return;
+
return trans.injector().get('User').read()
.then(() => {
try {
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/panel-collapsible/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/panel-collapsible/controller.js b/modules/web-console/frontend/app/components/panel-collapsible/controller.js
index 0b8b4da..65ec06a 100644
--- a/modules/web-console/frontend/app/components/panel-collapsible/controller.js
+++ b/modules/web-console/frontend/app/components/panel-collapsible/controller.js
@@ -33,20 +33,31 @@ export default class PanelCollapsible {
constructor($transclude) {
this.$transclude = $transclude;
}
+
toggle() {
if (this.opened)
this.close();
else
this.open();
}
+
open() {
- if (this.disabled) return;
+ if (this.disabled)
+ return;
+
this.opened = true;
- if (this.onOpen && this.opened) this.onOpen({});
+
+ if (this.onOpen && this.opened)
+ this.onOpen({});
}
+
close() {
- if (this.disabled) return;
+ if (this.disabled)
+ return;
+
this.opened = false;
- if (this.onClose && !this.opened) this.onClose({});
+
+ if (this.onClose && !this.opened)
+ this.onClose({});
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ui-grid-filters/directive.js b/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
index 6602c11..c9b84af 100644
--- a/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
+++ b/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
@@ -23,7 +23,8 @@ export default function uiGridFilters(uiGridConstants) {
require: 'uiGrid',
link: {
pre(scope, el, attr, gridApi) {
- if (!gridApi.grid.options.enableFiltering) return;
+ if (!gridApi.grid.options.enableFiltering)
+ return;
const applyMultiselectFilter = (cd) => {
cd.headerCellTemplate = template;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/components/ui-grid-filters/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ui-grid-filters/index.js b/modules/web-console/frontend/app/components/ui-grid-filters/index.js
index 0f05b77..2c60d89 100644
--- a/modules/web-console/frontend/app/components/ui-grid-filters/index.js
+++ b/modules/web-console/frontend/app/components/ui-grid-filters/index.js
@@ -27,7 +27,9 @@ export default angular
instance.$referenceElement = el;
instance.destroy = flow(instance.destroy, () => instance.$referenceElement = null);
instance.$applyPlacement = flow(instance.$applyPlacement, () => {
- if (!instance.$element) return;
+ if (!instance.$element)
+ return;
+
const refWidth = instance.$referenceElement[0].getBoundingClientRect().width;
const elWidth = instance.$element[0].getBoundingClientRect().width;
if (refWidth > elWidth) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 95b7ce4..c3efea5 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -1394,7 +1394,8 @@ export default class IgniteConfigurationGenerator {
.longProperty('rateTimeInterval');
}
- if (plcBean.isEmpty()) return;
+ if (plcBean.isEmpty())
+ return;
policies.push(plcBean);
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js b/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js
index 56aa82a..cecb49c 100644
--- a/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js
+++ b/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js
@@ -24,7 +24,9 @@ export default ['bsSelect', [() => {
const $render = ngModel.$render;
ngModel.$render = () => {
- if (scope.$destroyed) return;
+ if (scope.$destroyed)
+ return;
+
scope.$applyAsync(() => {
$render();
const value = ngModel.$viewValue;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/modules/form/validator/unique.directive.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/form/validator/unique.directive.js b/modules/web-console/frontend/app/modules/form/validator/unique.directive.js
index d32c565..318b8b6 100644
--- a/modules/web-console/frontend/app/modules/form/validator/unique.directive.js
+++ b/modules/web-console/frontend/app/modules/form/validator/unique.directive.js
@@ -42,12 +42,14 @@ class Controller {
if (!this.skip) {
// Return true in case if array not exist, array empty.
- if (!this.items || !this.items.length) return true;
+ if (!this.items || !this.items.length)
+ return true;
const idx = this.items.findIndex(matches);
// In case of new element check all items.
- if (isNew) return idx < 0;
+ if (isNew)
+ return idx < 0;
// Case for new component list editable.
const $index = this.listEditableTransclude
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/services/Caches.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/services/Caches.js b/modules/web-console/frontend/app/services/Caches.js
index add63f8..7df4bb6 100644
--- a/modules/web-console/frontend/app/services/Caches.js
+++ b/modules/web-console/frontend/app/services/Caches.js
@@ -165,7 +165,10 @@ export default class Caches {
maxMemorySize: {
min: (evictionPolicy) => {
const policy = evictionPolicy[evictionPolicy.kind];
- if (!policy) return true;
+
+ if (!policy)
+ return true;
+
const maxSize = policy.maxSize === null || policy.maxSize === void 0
? this.evictionPolicy.maxSize.default
: policy.maxSize;
@@ -177,7 +180,10 @@ export default class Caches {
maxSize: {
min: (evictionPolicy) => {
const policy = evictionPolicy[evictionPolicy.kind];
- if (!policy) return true;
+
+ if (!policy)
+ return true;
+
const maxMemorySize = policy.maxMemorySize === null || policy.maxMemorySize === void 0
? this.evictionPolicy.maxMemorySize.default
: policy.maxMemorySize;
[30/50] [abbrv] ignite git commit: IGNITE-8939 Catch and proper
propagate transaction string reprsentation - Fixes #4454.
Posted by av...@apache.org.
IGNITE-8939 Catch and proper propagate transaction string reprsentation - Fixes #4454.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/458480c5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/458480c5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/458480c5
Branch: refs/heads/ignite-8446
Commit: 458480c5b0520aa8e28935361a37ab49e1e65ff6
Parents: 3ea3a56
Author: Evgeny Stanilovskiy <es...@gridgain.com>
Authored: Wed Aug 1 14:05:58 2018 +0300
Committer: Dmitriy Pavlov <dp...@apache.org>
Committed: Wed Aug 1 14:05:58 2018 +0300
----------------------------------------------------------------------
.../cache/transactions/IgniteTxHandler.java | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/458480c5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 8f2dbc0..5b82333 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -23,6 +23,8 @@ import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -954,7 +956,23 @@ public class IgniteTxHandler {
}
}
catch (Throwable e) {
- U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e);
+ try {
+ U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e);
+ }
+ catch (Throwable e0) {
+ ClusterNode node0 = ctx.discovery().node(nodeId);
+
+ U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" +
+ CU.txString(tx) + ']', e);
+
+ U.error(log, "Failed to log message due to an error: ", e0);
+
+ if (node0 != null && (!node0.isClient() || node0.isLocal())) {
+ ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+
+ throw e;
+ }
+ }
if (tx != null) {
tx.commitError(e);
[32/50] [abbrv] ignite git commit: IGNITE-8073 Cache read metric is
calculated incorrectly in atomic cache.
Posted by av...@apache.org.
IGNITE-8073 Cache read metric is calculated incorrectly in atomic cache.
Signed-off-by: Anton Vinogradov <av...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/251c91bf
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/251c91bf
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/251c91bf
Branch: refs/heads/ignite-8446
Commit: 251c91bf45bf53652e7f837751fd138134a18b51
Parents: f1ecbbc
Author: voipp <al...@gmail.com>
Authored: Wed Aug 1 17:27:22 2018 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Wed Aug 1 17:27:22 2018 +0300
----------------------------------------------------------------------
.../distributed/near/GridNearGetFuture.java | 2 +-
.../GridCacheNearAtomicMetricsSelfTest.java | 36 ++++++++++++++++++++
2 files changed, 37 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/251c91bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index e530750..a101e7f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -498,7 +498,7 @@ public final class GridNearGetFuture<K, V> extends CacheDistributedGetFutureAdap
return saved;
}
- if (cctx.statisticsEnabled() && !skipVals && !affNode.isLocal())
+ if (cctx.statisticsEnabled() && !skipVals && !affNode.isLocal() && !isNear)
cache().metrics0().onRead(false);
LinkedHashMap<KeyCacheObject, Boolean> keys = mapped.get(affNode);
http://git-wip-us.apache.org/repos/asf/ignite/blob/251c91bf/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearAtomicMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearAtomicMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearAtomicMetricsSelfTest.java
index 70cebff..4d82c71 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearAtomicMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearAtomicMetricsSelfTest.java
@@ -17,8 +17,11 @@
package org.apache.ignite.internal.processors.cache.distributed.near;
+import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
/**
* Atomic cache metrics test.
@@ -32,4 +35,37 @@ public class GridCacheNearAtomicMetricsSelfTest extends GridCacheNearMetricsSelf
return ccfg;
}
+
+ /**
+ * Checks that enabled near cache does not affect metrics.
+ */
+ public void testNearCachePutRemoveGetMetrics() {
+ IgniteEx initiator = grid(0);
+
+ IgniteCache<Integer, Integer> cache0 = initiator.cache(DEFAULT_CACHE_NAME);
+
+ for (int i = 0; ; i++) {
+ if (!initiator.affinity(DEFAULT_CACHE_NAME).isPrimaryOrBackup(initiator.cluster().localNode(), i)) {
+ cache0.put(i, i);
+
+ GridCacheEntryEx nearEntry = near(cache0).peekEx(i);
+
+ assertTrue("Near cache doesn't contain the key", nearEntry.hasValue());
+
+ cache0.remove(i);
+
+ nearEntry = near(cache0).peekEx(i);
+
+ assertFalse("Near cache contains key which was deleted", nearEntry.hasValue());
+
+ cache0.get(i);
+
+ assertEquals(1, cache0.localMetrics().getCachePuts());
+ assertEquals(1, cache0.localMetrics().getCacheRemovals());
+ assertEquals(1, cache0.localMetrics().getCacheGets());
+
+ break;
+ }
+ }
+ }
}
[46/50] [abbrv] ignite git commit: IGNITE-8989 Web Console: Fixed
defaults for clientConnectorConfiguration.
Posted by av...@apache.org.
IGNITE-8989 Web Console: Fixed defaults for clientConnectorConfiguration.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/83ee6d3c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/83ee6d3c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/83ee6d3c
Branch: refs/heads/ignite-8446
Commit: 83ee6d3cd196127eeae3ae69cc0c586a87f95ca5
Parents: b718e44
Author: Vasiliy Sisko <vs...@gridgain.com>
Authored: Fri Aug 3 00:37:39 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Aug 3 00:37:39 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/app/services/Clusters.js | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/83ee6d3c/modules/web-console/frontend/app/services/Clusters.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/services/Clusters.js b/modules/web-console/frontend/app/services/Clusters.js
index c33c145..0228a77 100644
--- a/modules/web-console/frontend/app/services/Clusters.js
+++ b/modules/web-console/frontend/app/services/Clusters.js
@@ -188,7 +188,11 @@ export default class Clusters {
tcpNoDelay: true
},
clientConnectorConfiguration: {
- tcpNoDelay: true
+ tcpNoDelay: true,
+ jdbcEnabled: true,
+ odbcEnabled: true,
+ thinClientEnabled: true,
+ useIgniteSslContextFactory: true
},
space: void 0,
discovery: {
[02/50] [abbrv] ignite git commit: IGNITE-9101 Web Console: Fixed e2e
tests.
Posted by av...@apache.org.
IGNITE-9101 Web Console: Fixed e2e tests.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ec569048
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ec569048
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ec569048
Branch: refs/heads/ignite-8446
Commit: ec5690487f1fe55742c7ee6f05a3bd4590babef8
Parents: a61e1a9
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Tue Jul 31 10:02:01 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Jul 31 10:02:01 2018 +0700
----------------------------------------------------------------------
.../e2e/testcafe/components/pageConfiguration.js | 2 +-
.../configuration/clusterFormChangeDetection.js | 12 +++++++-----
.../page-models/PageConfigurationAdvancedCluster.js | 11 +++++++++++
3 files changed, 19 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec569048/modules/web-console/e2e/testcafe/components/pageConfiguration.js
----------------------------------------------------------------------
diff --git a/modules/web-console/e2e/testcafe/components/pageConfiguration.js b/modules/web-console/e2e/testcafe/components/pageConfiguration.js
index c364208..033412b 100644
--- a/modules/web-console/e2e/testcafe/components/pageConfiguration.js
+++ b/modules/web-console/e2e/testcafe/components/pageConfiguration.js
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-import {Selector} from 'testcafe'
+import {Selector} from 'testcafe';
export const basicNavButton = Selector('.tabs.tabs--blue a[ui-sref="base.configuration.edit.basic"]');
export const advancedNavButton = Selector('.tabs.tabs--blue a[ui-sref="base.configuration.edit.advanced.cluster"]');
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec569048/modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js
----------------------------------------------------------------------
diff --git a/modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js b/modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js
index a862172..693add4 100644
--- a/modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js
+++ b/modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js
@@ -36,21 +36,23 @@ fixture('Cluster configuration form change detection')
})
.after(dropTestDB);
-test('New cluster change detection', async(t) => {
+test.skip('New cluster change detection', async(t) => {
const overview = new PageConfigurationOverview();
const advanced = new PageConfigurationAdvancedCluster();
await t
.navigateTo(resolveUrl(`/configuration/overview`))
.click(overview.createClusterConfigButton)
- .click(advancedNavButton)
- .click(advanced.sections.connectorConfiguration.panel.heading);
+ .click(advancedNavButton);
- await scrollIntoView.with({dependencies: {el: advanced.sections.connectorConfiguration.panel.heading}})();
+ await t.click(advanced.sections.connectorConfiguration.panel.heading);
+
+ // IODO: Investigate why this code doesn't work in headless mode;
+ await scrollIntoView.with({dependencies: {el: advanced.sections.connectorConfiguration.inputs.enable.control}})();
await t
.click(advanced.sections.connectorConfiguration.inputs.enable.control)
.click(advanced.saveButton)
.click(pageAdvancedConfiguration.cachesNavButton)
- .expect(confirmation.body.exists).notOk(`Doesn't show changes confiramtion after saving new cluster`);
+ .expect(confirmation.body.exists).notOk(`Doesn't show changes confirmation after saving new cluster`);
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec569048/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js
----------------------------------------------------------------------
diff --git a/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js b/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js
index 1f1f559..e326100 100644
--- a/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js
+++ b/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js
@@ -16,10 +16,21 @@
*/
import {Selector, t} from 'testcafe';
+import {PanelCollapsible} from '../components/PanelCollapsible';
+import {FormField} from '../components/FormField';
export class PageConfigurationAdvancedCluster {
constructor() {
this.saveButton = Selector('.pc-form-actions-panel .btn-ignite').withText('Save');
+
+ this.sections = {
+ connectorConfiguration: {
+ panel: new PanelCollapsible('Connector configuration'),
+ inputs: {
+ enable: new FormField({id: 'restEnabledInput'})
+ }
+ }
+ };
}
async save() {
[40/50] [abbrv] ignite git commit: IGNITE-9111 Do not wait for
deactivation.
Posted by av...@apache.org.
IGNITE-9111 Do not wait for deactivation.
Signed-off-by: agura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8811a143
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8811a143
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8811a143
Branch: refs/heads/ignite-8446
Commit: 8811a143b525804905fd661b2c66d508b70e4697
Parents: d632eb6
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Thu Aug 2 15:51:17 2018 +0300
Committer: agura <ag...@apache.org>
Committed: Thu Aug 2 15:51:46 2018 +0300
----------------------------------------------------------------------
.../cluster/DiscoveryDataClusterState.java | 7 ---
.../cluster/GridClusterStateProcessor.java | 2 +-
.../IgniteClusterActivateDeactivateTest.java | 48 ++++++++++++++++++++
3 files changed, 49 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8811a143/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
index b6029a3..65d6088 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
@@ -173,13 +173,6 @@ public class DiscoveryDataClusterState implements Serializable {
}
/**
- * @return {@code True} if cluster active state change is in progress, {@code false} otherwise.
- */
- public boolean activeStateChanging() {
- return transition() && (prevState == null || (prevState.active != active));
- }
-
- /**
* @return State change exchange version.
*/
public AffinityTopologyVersion transitionTopologyVersion() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/8811a143/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index da0bbf6..8d2620f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -179,7 +179,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter implements I
assert globalState != null;
- if (globalState.transition() && globalState.activeStateChanging()) {
+ if (globalState.transition() && globalState.active()) {
Boolean transitionRes = globalState.transitionResult();
if (transitionRes != null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/8811a143/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 3b39bfb..4ff97ba 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -42,6 +42,7 @@ import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -53,6 +54,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
@@ -1176,6 +1178,52 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
}
/**
+ * @throws Exception If failed.
+ */
+ public void testClusterStateNotWaitForDeactivation() throws Exception {
+ testSpi = true;
+
+ final int nodes = 2;
+
+ IgniteEx crd = (IgniteEx) startGrids(nodes);
+
+ crd.cluster().active(true);
+
+ AffinityTopologyVersion curTopVer = crd.context().discovery().topologyVersionEx();
+
+ AffinityTopologyVersion deactivationTopVer = new AffinityTopologyVersion(
+ curTopVer.topologyVersion(),
+ curTopVer.minorTopologyVersion() + 1
+ );
+
+ for (int gridIdx = 0; gridIdx < nodes; gridIdx++) {
+ TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(grid(gridIdx));
+
+ blockExchangeSingleMessage(spi, deactivationTopVer);
+ }
+
+ IgniteInternalFuture deactivationFut = GridTestUtils.runAsync(() -> crd.cluster().active(false));
+
+ // Wait for deactivation start.
+ GridTestUtils.waitForCondition(() -> {
+ DiscoveryDataClusterState clusterState = crd.context().state().clusterState();
+
+ return clusterState.transition() && !clusterState.active();
+ }, getTestTimeout());
+
+ // Check that deactivation transition wait is not happened.
+ Assert.assertFalse(crd.context().state().publicApiActiveState(true));
+
+ for (int gridIdx = 0; gridIdx < nodes; gridIdx++) {
+ TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(grid(gridIdx));
+
+ spi.stopBlock();
+ }
+
+ deactivationFut.get();
+ }
+
+ /**
* @param exp If {@code true} there should be recorded messages.
*/
private void checkRecordedMessages(boolean exp) {
[29/50] [abbrv] ignite git commit: IGNITE-9154: Fixed conflict
version passed to events for ATOMIC cache. This closes #4463.
Posted by av...@apache.org.
IGNITE-9154: Fixed conflict version passed to events for ATOMIC cache. This closes #4463.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3ea3a560
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3ea3a560
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3ea3a560
Branch: refs/heads/ignite-8446
Commit: 3ea3a560847e36c6aa367e83f3ba3545b6a6d99a
Parents: 4439119
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Wed Aug 1 13:23:32 2018 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Wed Aug 1 13:23:32 2018 +0300
----------------------------------------------------------------------
.../internal/processors/cache/GridCacheMapEntry.java | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ea3a560/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 9f9b5c5..90d4c4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -1881,7 +1881,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
key,
evtNodeId,
null,
- newVer,
+ updateVer,
EVT_CACHE_OBJECT_READ,
evtOld, evtOld != null,
evtOld, evtOld != null,
@@ -1908,7 +1908,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
key,
evtNodeId,
null,
- newVer,
+ updateVer,
EVT_CACHE_OBJECT_PUT,
updateVal,
true,
@@ -1925,7 +1925,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
clearReaders();
- drReplicate(drType, null, newVer, topVer);
+ drReplicate(drType, null, updateVer, topVer);
recordNodeId(affNodeId, topVer);
@@ -1936,7 +1936,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
cctx.events().addEvent(partition(),
key,
evtNodeId,
- null, newVer,
+ null,
+ updateVer,
EVT_CACHE_OBJECT_REMOVED,
null, false,
evtOld, evtOld != null,
@@ -5046,7 +5047,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
}
else {
- newSysTtl = newTtl = conflictCtx.ttl();
+ newSysTtl = newTtl = conflictCtx.ttl();
newSysExpireTime = newExpireTime = conflictCtx.expireTime();
}
[03/50] [abbrv] ignite git commit: IGNITE-9102 Web Console: IE11
marked as outdated browser.
Posted by av...@apache.org.
IGNITE-9102 Web Console: IE11 marked as outdated browser.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b1539457
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b1539457
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b1539457
Branch: refs/heads/ignite-8446
Commit: b1539457ae6c3cfcfd803cf14e6c94ac9d7cd05c
Parents: ec56904
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Tue Jul 31 10:05:53 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Jul 31 10:05:53 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/app/browserUpdate/index.js | 8 +++++++-
modules/web-console/frontend/package.json | 2 +-
2 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b1539457/modules/web-console/frontend/app/browserUpdate/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/browserUpdate/index.js b/modules/web-console/frontend/app/browserUpdate/index.js
index 85df64e..799c4fc 100644
--- a/modules/web-console/frontend/app/browserUpdate/index.js
+++ b/modules/web-console/frontend/app/browserUpdate/index.js
@@ -20,7 +20,7 @@ import './style.scss';
browserUpdate({
notify: {
- i: 10,
+ i: 11,
f: '-18m',
s: 9,
c: '-18m',
@@ -30,5 +30,11 @@ browserUpdate({
l: 'en',
mobile: false,
api: 5,
+ text: `
+ <b>Outdated or unsupported browser detected.</b>
+ Web Console may work incorrectly. Please update to one of modern fully supported browser!
+ <a {up_but}>Update</a>
+ <a {ignore_but}>Ignore</a>
+ `,
reminder: 0
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/b1539457/modules/web-console/frontend/package.json
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 666a963..02acbe0 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -52,7 +52,7 @@
"angular-ui-validate": "1.2.3",
"angular1-async-filter": "1.1.0",
"brace": "0.10.0",
- "browser-update": "2.1.9",
+ "browser-update": "3.1.13",
"bson-objectid": "1.1.5",
"file-saver": "1.3.3",
"font-awesome": "4.7.0",
[48/50] [abbrv] ignite git commit: IGNITE-7650 Minor UI tweaks.
Posted by av...@apache.org.
IGNITE-7650 Minor UI tweaks.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f825dfc6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f825dfc6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f825dfc6
Branch: refs/heads/ignite-8446
Commit: f825dfc69b898d3810c7128fce095574cb668ba9
Parents: 00bbd8e
Author: Dmitriy Shabalin <dm...@gmail.com>
Authored: Fri Aug 3 15:26:24 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Aug 3 15:26:24 2018 +0700
----------------------------------------------------------------------
.../app/primitives/form-field/checkbox.pug | 2 +-
.../app/primitives/form-field/index.scss | 46 +++++++++++++++++++-
.../app/primitives/form-field/number.pug | 5 ++-
3 files changed, 49 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f825dfc6/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/primitives/form-field/checkbox.pug b/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
index 09596d4..b498cbd 100644
--- a/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
@@ -15,7 +15,7 @@
limitations under the License.
mixin form-field__checkbox({ label, model, name, disabled, required, tip })
- .form-field.form-field__checkbox
+ .form-field.form-field__checkbox(id=`{{ ${name} }}Field`)
+form-field__label({ label, name, required })
+form-field__tooltip({ title: tip, options: tipOpts })
http://git-wip-us.apache.org/repos/asf/ignite/blob/f825dfc6/modules/web-console/frontend/app/primitives/form-field/index.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/primitives/form-field/index.scss b/modules/web-console/frontend/app/primitives/form-field/index.scss
index 83dd55d..070dfe7 100644
--- a/modules/web-console/frontend/app/primitives/form-field/index.scss
+++ b/modules/web-console/frontend/app/primitives/form-field/index.scss
@@ -130,7 +130,7 @@
}
.ignite-form-field__errors {
color: $ignite-brand-primary;
- padding: 5px 10px 0px;
+ padding: 5px 10px 0;
line-height: 14px;
font-size: 12px;
clear: both;
@@ -215,6 +215,7 @@
box-sizing: border-box;
width: 100%;
max-width: initial;
+ min-width: 0;
height: 36px;
padding: 9px 10px;
margin-right: 0;
@@ -241,6 +242,13 @@
}
}
+ &--postfix::after {
+ content: attr(data-postfix);
+ display: inline-flex;
+ place-self: center;
+ margin-left: 10px;
+ }
+
// Added right offset to appearance of input for invalid password
& > input[type='email'].ng-invalid.ng-touched,
& > input[type='text'].ng-invalid.ng-touched,
@@ -261,6 +269,8 @@
right: 0;
bottom: 0;
+ display: flex;
+
[ng-message] {
// TODO: remove after replace all fields to new
overflow: visible !important;
@@ -268,6 +278,12 @@
}
}
+ &__control--postfix + &__errors::after {
+ content: attr(data-postfix);
+ margin-left: 10px;
+ visibility: hidden;
+ }
+
&__error {
z-index: 2;
position: relative;
@@ -328,6 +344,7 @@
width: auto;
height: auto;
margin: 0;
+ border-radius: 0;
}
}
@@ -382,3 +399,30 @@
height: 36px;
}
}
+
+.form-fieldset {
+ padding: 10px;
+
+ border: 1px solid hsla(0,0%,77%,.5);
+ border-radius: 4px;
+
+ legend {
+ width: auto;
+ margin: 0 -5px;
+ padding: 0 5px;
+
+ border: 0;
+
+ color: #393939;
+ font-size: 14px;
+ line-height: 1.42857;
+ }
+
+ legend + * {
+ margin-top: 0 !important;
+ }
+
+ & > *:last-child {
+ margin-bottom: 0 !important;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f825dfc6/modules/web-console/frontend/app/primitives/form-field/number.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/primitives/form-field/number.pug b/modules/web-console/frontend/app/primitives/form-field/number.pug
index e5f1454..11f8e22 100644
--- a/modules/web-console/frontend/app/primitives/form-field/number.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/number.pug
@@ -14,14 +14,14 @@
See the License for the specific language governing permissions and
limitations under the License.
-mixin form-field__number({ label, model, name, disabled, required, placeholder, tip, min, max, step })
+mixin form-field__number({ label, model, name, disabled, required, placeholder, tip, min, max, step, postfix })
-var errLbl = label.substring(0, label.length - 1)
.form-field
+form-field__label({ label, name, required })
+form-field__tooltip({ title: tip, options: tipOpts })
- .form-field__control
+ .form-field__control(class=postfix && 'form-field__control--postfix' data-postfix=postfix)
- attributes.type = 'number'
- attributes.min = min ? min : '0'
- attributes.max = max ? max : '{{ Number.MAX_VALUE }}'
@@ -29,6 +29,7 @@ mixin form-field__number({ label, model, name, disabled, required, placeholder,
+form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes)
.form-field__errors(
+ data-postfix=postfix
ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}`
)
if block
[04/50] [abbrv] ignite git commit: IGNITE-9103 Web Console: Fixed
select menu component with items that contains html.
Posted by av...@apache.org.
IGNITE-9103 Web Console: Fixed select menu component with items that contains html.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7c4705c0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7c4705c0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7c4705c0
Branch: refs/heads/ignite-8446
Commit: 7c4705c0b9a248608fadccd874fcccc1e7378ea7
Parents: b153945
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Tue Jul 31 10:14:10 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Jul 31 10:14:10 2018 +0700
----------------------------------------------------------------------
.../app/components/bs-select-menu/index.js | 6 ++++--
.../components/bs-select-menu/strip.filter.js | 22 ++++++++++++++++++++
.../app/components/bs-select-menu/template.pug | 2 +-
.../transcludeToBody.directive.js | 2 +-
4 files changed, 28 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7c4705c0/modules/web-console/frontend/app/components/bs-select-menu/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/index.js b/modules/web-console/frontend/app/components/bs-select-menu/index.js
index a9bafb2..0f2a258 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/index.js
+++ b/modules/web-console/frontend/app/components/bs-select-menu/index.js
@@ -18,9 +18,11 @@
import angular from 'angular';
import directive from './directive';
-import {directive as transcludeToBody} from './transcludeToBody.directive';
+import {default as transcludeToBody} from './transcludeToBody.directive';
+import stripFilter from './strip.filter';
export default angular
.module('ignite-console.bs-select-menu', [])
.directive('bssmTranscludeToBody', transcludeToBody)
- .directive('bsSelectMenu', directive);
+ .directive('bsSelectMenu', directive)
+ .filter('bsSelectStrip', stripFilter);
http://git-wip-us.apache.org/repos/asf/ignite/blob/7c4705c0/modules/web-console/frontend/app/components/bs-select-menu/strip.filter.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/strip.filter.js b/modules/web-console/frontend/app/components/bs-select-menu/strip.filter.js
new file mode 100644
index 0000000..4dbe58b
--- /dev/null
+++ b/modules/web-console/frontend/app/components/bs-select-menu/strip.filter.js
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default function() {
+ return function(val) {
+ return val.replace(/(<\/?\w+>)/igm, '');
+ };
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7c4705c0/modules/web-console/frontend/app/components/bs-select-menu/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/template.pug b/modules/web-console/frontend/app/components/bs-select-menu/template.pug
index 4ffa144..7cd2195 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/template.pug
+++ b/modules/web-console/frontend/app/components/bs-select-menu/template.pug
@@ -37,7 +37,7 @@ ul.bs-select-menu(
ng-click='$select($index, $event); $event.stopPropagation();'
ng-class=`{ 'bssm-item-button__active': $isActive($index) }`
data-placement='right auto'
- title='{{ ::match.label }}'
+ title='{{ ::match.label | bsSelectStrip }}'
)
img.bssm-active-indicator.icon-left(
ng-src='{{ $isActive($index) ? "/images/checkbox-active.svg" : "/images/checkbox.svg" }}'
http://git-wip-us.apache.org/repos/asf/ignite/blob/7c4705c0/modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js b/modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js
index b415719..de4fbfa 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js
+++ b/modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js
@@ -40,7 +40,7 @@ class Controller {
}
}
-export function directive() {
+export default function directive() {
return {
restrict: 'E',
transclude: true,
[21/50] [abbrv] ignite git commit: Merge remote-tracking branch
'origin/master'
Posted by av...@apache.org.
Merge remote-tracking branch 'origin/master'
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1f064206
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1f064206
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1f064206
Branch: refs/heads/ignite-8446
Commit: 1f064206343f19dc84a9ceb66cc5544ebbe7fcae
Parents: f041175 9e884e5
Author: devozerov <pp...@gmail.com>
Authored: Tue Jul 31 17:16:36 2018 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Tue Jul 31 17:16:36 2018 +0300
----------------------------------------------------------------------
examples/pom.xml | 21 -
.../cluster/TensorFlowClusterExample.java | 118 ------
.../org/apache/ignite/cache/CacheMetrics.java | 77 ++++
.../internal/commandline/CommandHandler.java | 162 ++++---
.../commandline/cache/CacheArguments.java | 34 ++
.../cache/CacheClusterMetricsMXBeanImpl.java | 55 +++
.../cache/CacheLocalMetricsMXBeanImpl.java | 55 +++
.../processors/cache/CacheMetricsImpl.java | 206 +++++++++
.../processors/cache/CacheMetricsSnapshot.java | 157 +++++++
.../processors/cache/GridCacheAdapter.java | 67 ++-
.../processors/cache/GridCacheEntryEx.java | 8 +-
.../processors/cache/GridCacheMapEntry.java | 115 +++--
.../cache/GridCacheUpdateAtomicResult.java | 15 +-
.../distributed/dht/GridDhtTxPrepareFuture.java | 5 +
.../GridDhtAtomicAbstractUpdateFuture.java | 8 +-
.../GridDhtAtomicAbstractUpdateRequest.java | 16 +-
.../dht/atomic/GridDhtAtomicCache.java | 63 ++-
.../GridDhtAtomicSingleUpdateRequest.java | 11 +-
.../dht/atomic/GridDhtAtomicUpdateRequest.java | 5 +-
.../distributed/near/GridNearAtomicCache.java | 14 +-
.../local/atomic/GridLocalAtomicCache.java | 116 ++++--
.../cache/transactions/IgniteTxAdapter.java | 2 +
.../cache/verify/IdleVerifyDumpResult.java | 73 ++++
.../cache/verify/IdleVerifyResultV2.java | 85 +++-
.../verify/VerifyBackupPartitionsDumpTask.java | 230 ++++++++++
.../platform/cache/PlatformCache.java | 11 +
.../visor/verify/VisorIdleVerifyDumpTask.java | 37 ++
.../verify/VisorIdleVerifyDumpTaskArg.java | 73 ++++
.../visor/verify/VisorIdleVerifyJob.java | 83 ++++
.../resources/META-INF/classnames.properties | 2 +
.../cache/GridCacheAbstractMetricsSelfTest.java | 417 +++++++++++++++++++
.../processors/cache/GridCacheTestEntryEx.java | 8 +-
.../GridCacheNearAtomicMetricsSelfTest.java | 35 ++
.../near/GridCacheNearMetricsSelfTest.java | 388 +++++++++++++++++
.../platform/PlatformCacheWriteMetricsTask.java | 55 +++
.../testframework/junits/GridAbstractTest.java | 95 +++--
.../ignite/testsuites/IgniteBasicTestSuite.java | 12 +-
.../IgniteBasicWithPersistenceTestSuite.java | 67 +++
...IgniteBinaryObjectsComputeGridTestSuite.java | 3 +
.../IgniteCacheEvictionSelfTestSuite.java | 3 -
.../IgniteCacheFullApiSelfTestSuite.java | 3 +
.../IgniteCacheMetricsSelfTestSuite.java | 6 +-
.../ignite/testsuites/IgniteCacheTestSuite.java | 8 +-
.../testsuites/IgniteCacheTestSuite2.java | 5 +-
.../testsuites/IgniteCacheTestSuite3.java | 7 +-
.../testsuites/IgniteCacheTestSuite4.java | 5 +-
.../testsuites/IgniteCacheTestSuite5.java | 6 +-
.../testsuites/IgniteCacheTestSuite6.java | 28 +-
.../testsuites/IgniteCacheTestSuite7.java | 53 ++-
.../testsuites/IgniteComputeGridTestSuite.java | 2 -
.../testsuites/IgniteKernalSelfTestSuite.java | 7 +-
.../IgniteMarshallerSelfTestSuite.java | 5 +-
.../testsuites/IgniteUtilSelfTestSuite.java | 7 +-
.../ignite/util/GridCommandHandlerTest.java | 178 +++++++-
.../GridInternalTaskUnusedWalSegmentsTest.java | 27 +-
...acheWithIndexingAndPersistenceTestSuite.java | 38 ++
.../IgniteCacheWithIndexingTestSuite.java | 6 +-
.../Cache/CacheMetricsTest.cs | 11 +
.../Apache.Ignite.Core/Cache/ICacheMetrics.cs | 88 ++++
.../Impl/Cache/CacheMetricsImpl.cs | 79 +++-
modules/tensorflow/README.txt | 55 ++-
modules/tensorflow/pom.xml | 56 ++-
modules/tensorflow/src/main/assembly/zip.xml | 56 +++
.../cluster/TensorFlowClusterGateway.java | 34 +-
.../TensorFlowClusterGatewayManager.java | 95 ++++-
.../cluster/TensorFlowClusterMaintainer.java | 105 +++--
.../cluster/TensorFlowClusterManager.java | 268 ++++++++----
.../cluster/TensorFlowJobArchive.java | 65 +++
.../cluster/spec/TensorFlowClusterSpec.java | 39 ++
.../spec/TensorFlowServerAddressSpec.java | 14 +
.../tfrunning/TensorFlowServerManager.java | 112 +----
.../TensorFlowServerScriptFormatter.java | 62 +++
.../cluster/util/ClusterPortManager.java | 168 ++++++--
.../cluster/util/TensorFlowChiefRunner.java | 80 ++++
.../cluster/util/TensorFlowClusterResolver.java | 92 ++--
.../util/TensorFlowUserScriptRunner.java | 236 +++++++++++
.../ignite/tensorflow/core/ProcessManager.java | 5 +-
.../tensorflow/core/ProcessManagerWrapper.java | 5 +-
.../longrunning/LongRunningProcessManager.java | 19 +-
.../task/LongRunningProcessStartTask.java | 4 +-
.../core/nativerunning/NativeProcess.java | 6 +-
.../nativerunning/NativeProcessManager.java | 12 +-
.../task/NativeProcessStartTask.java | 81 +---
.../PythonProcessBuilderSupplier.java | 57 +++
.../pythonrunning/PythonProcessManager.java | 35 +-
.../core/util/AsyncNativeProcessRunner.java | 107 +++++
.../core/util/NativeProcessRunner.java | 133 ++++++
.../tensorflow/submitter/JobSubmitter.java | 35 ++
.../submitter/command/AbstractCommand.java | 55 +++
.../submitter/command/AttachCommand.java | 51 +++
.../tensorflow/submitter/command/PsCommand.java | 47 +++
.../submitter/command/RootCommand.java | 42 ++
.../submitter/command/StartCommand.java | 205 +++++++++
.../submitter/command/StopCommand.java | 50 +++
.../submitter/command/package-info.java | 22 +
.../tensorflow/submitter/package-info.java | 23 +
.../tensorflow/util/SerializableConsumer.java | 29 ++
.../tensorflow/util/SerializableSupplier.java | 29 ++
.../ignite/tensorflow/util/package-info.java | 22 +
modules/tensorflow/src/main/sh/ignite-tf.sh | 19 +
modules/tensorflow/src/main/sh/logback.xml | 36 ++
.../LongRunningProcessManagerTest.java | 10 +-
102 files changed, 5346 insertions(+), 906 deletions(-)
----------------------------------------------------------------------
[08/50] [abbrv] ignite git commit: IGNITE-4010 Web Console: Added CSS
constants.
Posted by av...@apache.org.
IGNITE-4010 Web Console: Added CSS constants.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c2383e92
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c2383e92
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c2383e92
Branch: refs/heads/ignite-8446
Commit: c2383e923c79fdddd04dcd55f322d10bd7e1f975
Parents: 66d69c2
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Tue Jul 31 11:17:32 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Jul 31 11:17:32 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/public/stylesheets/variables.scss | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c2383e92/modules/web-console/frontend/public/stylesheets/variables.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/public/stylesheets/variables.scss b/modules/web-console/frontend/public/stylesheets/variables.scss
index 22e4b9b..bc0c2d5 100644
--- a/modules/web-console/frontend/public/stylesheets/variables.scss
+++ b/modules/web-console/frontend/public/stylesheets/variables.scss
@@ -32,4 +32,8 @@ $ignite-button-border-radius: 4px;
// Color palette
$ignite-brand-primary: #ee2b27; // Red
-$ignite-brand-success: #0067b9; // Blue
\ No newline at end of file
+$ignite-brand-success: #0067b9; // Blue
+
+// Color for statuses
+$ignite-status-active: #417505;
+$ignite-status-inactive: #ee2b27;
\ No newline at end of file
[13/50] [abbrv] ignite git commit: IGNITE-9136 Web Console: Do not
show (de)activate switcher for clusters older than v2.0.0.
Posted by av...@apache.org.
IGNITE-9136 Web Console: Do not show (de)activate switcher for clusters older than v2.0.0.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bcda7a1f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bcda7a1f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bcda7a1f
Branch: refs/heads/ignite-8446
Commit: bcda7a1f584af3f8c3d1db3020ddc952dd4bb918
Parents: bea6fe3
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Tue Jul 31 18:41:01 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Jul 31 18:41:01 2018 +0700
----------------------------------------------------------------------
.../app/components/cluster-selector/controller.js | 13 ++++++++++---
.../app/components/cluster-selector/template.pug | 8 ++++----
2 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bcda7a1f/modules/web-console/frontend/app/components/cluster-selector/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/cluster-selector/controller.js b/modules/web-console/frontend/app/components/cluster-selector/controller.js
index 4aebd62..63421b1 100644
--- a/modules/web-console/frontend/app/components/cluster-selector/controller.js
+++ b/modules/web-console/frontend/app/components/cluster-selector/controller.js
@@ -21,15 +21,18 @@ import { BehaviorSubject } from 'rxjs/BehaviorSubject';
import 'rxjs/add/operator/combineLatest';
export default class {
- static $inject = ['AgentManager', 'IgniteConfirm'];
+ static $inject = ['AgentManager', 'IgniteConfirm', 'IgniteVersion'];
/**
* @param agentMgr Agent manager.
- * @param Confirm Confirmation service.
+ * @param Confirm Confirmation service.
+ * @param Version Version check service.
*/
- constructor(agentMgr, Confirm) {
+ constructor(agentMgr, Confirm, Version) {
this.agentMgr = agentMgr;
this.Confirm = Confirm;
+ this.Version = Version;
+
this.clusters = [];
this.isDemo = agentMgr.isDemoMode();
this._inProgressSubject = new BehaviorSubject(false);
@@ -61,6 +64,10 @@ export default class {
this.agentMgr.switchCluster(this.cluster);
}
+ isChangeStateAvailable() {
+ return !this.isDemo && this.cluster && this.Version.since(this.cluster.clusterVersion, '2.0.0');
+ }
+
toggle($event) {
$event.preventDefault();
http://git-wip-us.apache.org/repos/asf/ignite/blob/bcda7a1f/modules/web-console/frontend/app/components/cluster-selector/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/cluster-selector/template.pug b/modules/web-console/frontend/app/components/cluster-selector/template.pug
index d397164..f1e80db 100644
--- a/modules/web-console/frontend/app/components/cluster-selector/template.pug
+++ b/modules/web-console/frontend/app/components/cluster-selector/template.pug
@@ -63,15 +63,15 @@ svg(
ignite-icon='info'
bs-tooltip=''
data-title='Multi-Cluster Support<br/>\
- <a href="https://apacheignite-tools.readme.io/docs/multi-cluster-support" target="_blank">More info</a>'
+ <a href="https://apacheignite-tools.readme.io/docs/multi-cluster-support" target="_blank">More info</a>'
data-placement='bottom'
)
-.cluster-selector--state(ng-if='!$ctrl.isDemo && $ctrl.cluster')
+.cluster-selector--state(ng-if='$ctrl.isChangeStateAvailable()')
| Cluster {{ $ctrl.cluster.active ? 'active' : 'inactive' }}
+switcher()(
- ng-if='!$ctrl.isDemo && $ctrl.cluster'
+ ng-if='$ctrl.isChangeStateAvailable()'
ng-click='$ctrl.toggle($event)'
ng-checked='$ctrl.cluster.active'
ng-disabled='$ctrl.inProgress'
@@ -80,5 +80,5 @@ svg(
is-in-progress='{{ $ctrl.inProgress }}'
)
-div(ng-if='$ctrl.inProgress')
+div(ng-if='$ctrl.inProgress && $ctrl.isChangeStateAvailable()')
| {{ !$ctrl.cluster.active ? 'Activating...' : 'Deactivating...' }}
[07/50] [abbrv] ignite git commit: IGNITE-9106 Web Console: Added
bytes filter.
Posted by av...@apache.org.
IGNITE-9106 Web Console: Added bytes filter.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/66d69c2b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/66d69c2b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/66d69c2b
Branch: refs/heads/ignite-8446
Commit: 66d69c2b328f200c4f8955f439b2c63dd8b6c415
Parents: 4d9f450
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Tue Jul 31 10:28:13 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Jul 31 10:28:13 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/app/app.js | 2 ++
.../frontend/app/filters/bytes.filter.js | 34 ++++++++++++++++++
.../frontend/app/filters/bytes.filter.spec.js | 36 ++++++++++++++++++++
3 files changed, 72 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/66d69c2b/modules/web-console/frontend/app/app.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index e807319..1f81f86 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -100,6 +100,7 @@ import AngularStrapSelect from './services/AngularStrapSelect.decorator';
// Filters.
import byName from './filters/byName.filter';
+import bytes from './filters/bytes.filter';
import defaultName from './filters/default-name.filter';
import domainsValidation from './filters/domainsValidation.filter';
import duration from './filters/duration.filter';
@@ -305,6 +306,7 @@ export default angular.module('ignite-console', [
.service('Models', Models)
// Filters.
.filter('byName', byName)
+.filter('bytes', bytes)
.filter('defaultName', defaultName)
.filter('domainsValidation', domainsValidation)
.filter('duration', duration)
http://git-wip-us.apache.org/repos/asf/ignite/blob/66d69c2b/modules/web-console/frontend/app/filters/bytes.filter.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/filters/bytes.filter.js b/modules/web-console/frontend/app/filters/bytes.filter.js
new file mode 100644
index 0000000..7576dad
--- /dev/null
+++ b/modules/web-console/frontend/app/filters/bytes.filter.js
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default () => {
+ return (bytes, precision) => {
+ if (bytes === 0)
+ return '0 bytes';
+
+ if (isNaN(parseFloat(bytes)) || !isFinite(bytes))
+ return '-';
+
+ if (typeof precision === 'undefined')
+ precision = 1;
+
+ const units = ['bytes', 'kB', 'MB', 'GB', 'TB'];
+ const number = Math.floor(Math.log(bytes) / Math.log(1024));
+
+ return (bytes / Math.pow(1024, Math.floor(number))).toFixed(precision) + ' ' + units[number];
+ };
+};
http://git-wip-us.apache.org/repos/asf/ignite/blob/66d69c2b/modules/web-console/frontend/app/filters/bytes.filter.spec.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/filters/bytes.filter.spec.js b/modules/web-console/frontend/app/filters/bytes.filter.spec.js
new file mode 100644
index 0000000..884e43e
--- /dev/null
+++ b/modules/web-console/frontend/app/filters/bytes.filter.spec.js
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import bytesFilter from './bytes.filter';
+
+import { suite, test } from 'mocha';
+import { assert } from 'chai';
+
+const bytesFilterInstance = bytesFilter();
+
+suite('bytes filter', () => {
+ test('bytes filter', () => {
+ assert.equal(bytesFilterInstance(0), '0 bytes');
+ assert.equal(bytesFilterInstance(1000), '1000.0 bytes');
+ assert.equal(bytesFilterInstance(1024), '1.0 kB');
+ assert.equal(bytesFilterInstance(5000), '4.9 kB');
+ assert.equal(bytesFilterInstance(1048576), '1.0 MB');
+ assert.equal(bytesFilterInstance(104857600), '100.0 MB');
+ assert.equal(bytesFilterInstance(1073741824), '1.0 GB');
+ assert.equal(bytesFilterInstance(1099511627776), '1.0 TB');
+ });
+});
[27/50] [abbrv] ignite git commit: Minor fix to JavaDocs.
Posted by av...@apache.org.
Minor fix to JavaDocs.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ff891938
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ff891938
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ff891938
Branch: refs/heads/ignite-8446
Commit: ff891938329e27d2431d74bab53deb5812513cb9
Parents: 6cc633b
Author: devozerov <pp...@gmail.com>
Authored: Wed Aug 1 10:49:23 2018 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Wed Aug 1 10:49:23 2018 +0300
----------------------------------------------------------------------
.../processors/query/h2/twostep/GridReduceQueryExecutor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ff891938/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 39b2bbc..65aeae9 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -113,7 +113,7 @@ import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySpl
* Reduce query executor.
*/
public class GridReduceQueryExecutor {
- /** Fail query after 10 seconds of unsuccessful attempts to reserve partitions. */
+ /** Default retry timeout. */
public static final long DFLT_RETRY_TIMEOUT = 30_000L;
/** */
[47/50] [abbrv] ignite git commit: IGNITE-9163 Web console: Fixed
incorrect cluster state for clusters of version 1.x.
Posted by av...@apache.org.
IGNITE-9163 Web console: Fixed incorrect cluster state for clusters of version 1.x.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/00bbd8ea
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/00bbd8ea
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/00bbd8ea
Branch: refs/heads/ignite-8446
Commit: 00bbd8eae84a0b37d585b5b384729bf7b3440a66
Parents: 83ee6d3
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Fri Aug 3 14:32:03 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Aug 3 14:32:03 2018 +0700
----------------------------------------------------------------------
.../apache/ignite/console/agent/handlers/ClusterListener.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/00bbd8ea/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
----------------------------------------------------------------------
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
index 0c7560c..1c93798 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
@@ -66,6 +66,9 @@ public class ClusterListener implements AutoCloseable {
private static final IgniteLogger log = new Slf4jLogger(LoggerFactory.getLogger(ClusterListener.class));
/** */
+ private static final IgniteProductVersion IGNITE_2_0 = IgniteProductVersion.fromString("2.0.0");
+
+ /** */
private static final IgniteProductVersion IGNITE_2_1 = IgniteProductVersion.fromString("2.1.0");
/** */
@@ -418,6 +421,10 @@ public class ClusterListener implements AutoCloseable {
* @throws IOException If failed to collect cluster active state.
*/
public boolean active(IgniteProductVersion ver, UUID nid) throws IOException {
+ // 1.x clusters are always active.
+ if (ver.compareTo(IGNITE_2_0) < 0)
+ return true;
+
Map<String, Object> params = U.newHashMap(10);
boolean v23 = ver.compareTo(IGNITE_2_3) >= 0;
[44/50] [abbrv] ignite git commit: IGNITE-9073 Use jackson2
dependency instead of jackson1 and remove unused dependencies - Fixes #4444.
Posted by av...@apache.org.
IGNITE-9073 Use jackson2 dependency instead of jackson1 and remove unused dependencies - Fixes #4444.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/24bd54a7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/24bd54a7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/24bd54a7
Branch: refs/heads/ignite-8446
Commit: 24bd54a7c38a0f1e73dbfc32bf33914c76c19d9e
Parents: 10b5914
Author: zzzadruga <zz...@gmail.com>
Authored: Thu Aug 2 18:20:02 2018 +0300
Committer: Dmitriy Pavlov <dp...@apache.org>
Committed: Thu Aug 2 18:20:02 2018 +0300
----------------------------------------------------------------------
.../src/test/config/aop/aspectj/META-INF/aop.xml | 4 ++--
modules/aws/pom.xml | 18 ------------------
modules/gce/pom.xml | 2 +-
modules/kubernetes/pom.xml | 8 ++++----
.../TcpDiscoveryKubernetesIpFinder.java | 4 ++--
.../osgi-karaf/src/main/resources/features.xml | 10 +++++-----
modules/rest-http/pom.xml | 6 +++---
modules/spark/pom.xml | 6 ------
modules/web-console/web-agent/pom.xml | 2 +-
modules/zookeeper/pom.xml | 10 ++--------
.../zk/TcpDiscoveryZookeeperIpFinder.java | 2 +-
parent/pom.xml | 3 +--
12 files changed, 22 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd54a7/modules/aop/src/test/config/aop/aspectj/META-INF/aop.xml
----------------------------------------------------------------------
diff --git a/modules/aop/src/test/config/aop/aspectj/META-INF/aop.xml b/modules/aop/src/test/config/aop/aspectj/META-INF/aop.xml
index 79417da..8741bd1 100644
--- a/modules/aop/src/test/config/aop/aspectj/META-INF/aop.xml
+++ b/modules/aop/src/test/config/aop/aspectj/META-INF/aop.xml
@@ -139,7 +139,7 @@
<exclude within="com.google..*"/>
<!-- Jackson libs. -->
- <exclude within="org.codehaus.jackson..*"/>
+ <exclude within="com.fasterxml.jackson.core..*"/>
<!-- JLine libs. -->
<exclude within="org.fusesource..*"/>
@@ -285,7 +285,7 @@
<exclude within="com.google..*"/>
<!-- Jackson libs. -->
- <exclude within="org.codehaus.jackson..*"/>
+ <exclude within="com.fasterxml.jackson.core..*"/>
<!-- JLine libs. -->
<exclude within="org.fusesource..*"/>
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd54a7/modules/aws/pom.xml
----------------------------------------------------------------------
diff --git a/modules/aws/pom.xml b/modules/aws/pom.xml
index 914d864..e50b6df 100644
--- a/modules/aws/pom.xml
+++ b/modules/aws/pom.xml
@@ -72,24 +72,6 @@
</dependency>
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>${jackson2.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>${jackson2.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson2.version}</version>
- </dependency>
-
- <dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.8.1</version>
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd54a7/modules/gce/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gce/pom.xml b/modules/gce/pom.xml
index a8b9e4d..7b66c2a 100644
--- a/modules/gce/pom.xml
+++ b/modules/gce/pom.xml
@@ -62,7 +62,7 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
- <version>${jackson2.version}</version>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd54a7/modules/kubernetes/pom.xml
----------------------------------------------------------------------
diff --git a/modules/kubernetes/pom.xml b/modules/kubernetes/pom.xml
index f23c479..83fafd0 100644
--- a/modules/kubernetes/pom.xml
+++ b/modules/kubernetes/pom.xml
@@ -42,14 +42,14 @@
</dependency>
<dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd54a7/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
----------------------------------------------------------------------
diff --git a/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
index 53b6df6..51b68df 100644
--- a/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
+++ b/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java
@@ -30,6 +30,8 @@ import java.util.Collection;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.ObjectMapper;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
@@ -43,8 +45,6 @@ import org.apache.ignite.resources.LoggerResource;
import org.apache.ignite.spi.IgniteSpiException;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.ObjectMapper;
/**
* IP finder for automatic lookup of Ignite nodes running in Kubernetes environment. All Ignite nodes have to deployed
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd54a7/modules/osgi-karaf/src/main/resources/features.xml
----------------------------------------------------------------------
diff --git a/modules/osgi-karaf/src/main/resources/features.xml b/modules/osgi-karaf/src/main/resources/features.xml
index b4c5299..3e3989e 100644
--- a/modules/osgi-karaf/src/main/resources/features.xml
+++ b/modules/osgi-karaf/src/main/resources/features.xml
@@ -208,9 +208,9 @@
install -s mvn:xom/xom/1.2.5]]>
</details>
<feature dependency="true">http</feature>
- <bundle start="true" dependency="true">mvn:com.fasterxml.jackson.core/jackson-core/${jackson2.version}</bundle>
- <bundle start="true" dependency="true">mvn:com.fasterxml.jackson.core/jackson-databind/${jackson2.version}</bundle>
- <bundle start="true" dependency="true">mvn:com.fasterxml.jackson.core/jackson-annotations/${jackson2.version}</bundle>
+ <bundle start="true" dependency="true">mvn:com.fasterxml.jackson.core/jackson-core/${jackson.version}</bundle>
+ <bundle start="true" dependency="true">mvn:com.fasterxml.jackson.core/jackson-databind/${jackson.version}</bundle>
+ <bundle start="true" dependency="true">mvn:com.fasterxml.jackson.core/jackson-annotations/${jackson.version}</bundle>
<bundle start="true" dependency="true">mvn:commons-lang/commons-lang/${commons.lang.version}</bundle>
<bundle start="true" dependency="true">mvn:commons-collections/commons-collections/${commons.collections.version}</bundle>
<bundle start="true">mvn:org.apache.ignite/ignite-rest-http/${project.version}</bundle>
@@ -324,8 +324,8 @@
</details>
<bundle start="true" dependency="true">mvn:org.apache.zookeeper/zookeeper/${zookeeper.version}</bundle>
<bundle start="true" dependency="true">mvn:com.google.guava/guava/${guava16.version}</bundle>
- <bundle start="true" dependency="true">mvn:org.codehaus.jackson/jackson-core-asl/${jackson.version}</bundle>
- <bundle start="true" dependency="true">mvn:org.codehaus.jackson/jackson-mapper-asl/${jackson.version}</bundle>
+ <bundle start="true" dependency="true">mvn:com.fasterxml.jackson.core/jackson-core/${jackson.version}</bundle>
+ <bundle start="true" dependency="true">mvn:com.fasterxml.jackson.core/jackson-databind/${jackson.version}</bundle>
<bundle start="true" dependency="true">mvn:org.apache.curator/curator-client/${curator.version}</bundle>
<bundle start="true" dependency="true">mvn:org.apache.curator/curator-framework/${curator.version}</bundle>
<bundle start="true" dependency="true">mvn:org.apache.curator/curator-recipes/${curator.version}</bundle>
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd54a7/modules/rest-http/pom.xml
----------------------------------------------------------------------
diff --git a/modules/rest-http/pom.xml b/modules/rest-http/pom.xml
index 3772f8f..8cc31da 100644
--- a/modules/rest-http/pom.xml
+++ b/modules/rest-http/pom.xml
@@ -99,19 +99,19 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
- <version>${jackson2.version}</version>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
- <version>${jackson2.version}</version>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>${jackson2.version}</version>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd54a7/modules/spark/pom.xml
----------------------------------------------------------------------
diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml
index ff4db47..e8d4c2f 100644
--- a/modules/spark/pom.xml
+++ b/modules/spark/pom.xml
@@ -124,12 +124,6 @@
</dependency>
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>${jackson2.version}</version>
- </dependency>
-
- <dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-core_2.11</artifactId>
<version>3.5.0</version>
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd54a7/modules/web-console/web-agent/pom.xml
----------------------------------------------------------------------
diff --git a/modules/web-console/web-agent/pom.xml b/modules/web-console/web-agent/pom.xml
index 10f7965..06778c3 100644
--- a/modules/web-console/web-agent/pom.xml
+++ b/modules/web-console/web-agent/pom.xml
@@ -49,7 +49,7 @@
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-json-org</artifactId>
- <version>${jackson2.version}</version>
+ <version>${jackson.version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd54a7/modules/zookeeper/pom.xml
----------------------------------------------------------------------
diff --git a/modules/zookeeper/pom.xml b/modules/zookeeper/pom.xml
index 0fb5ddd..9c42e47 100644
--- a/modules/zookeeper/pom.xml
+++ b/modules/zookeeper/pom.xml
@@ -72,14 +72,8 @@
</dependency>
<dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- <version>${jackson.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd54a7/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/TcpDiscoveryZookeeperIpFinder.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/TcpDiscoveryZookeeperIpFinder.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/TcpDiscoveryZookeeperIpFinder.java
index f96a762..31d118b 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/TcpDiscoveryZookeeperIpFinder.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/TcpDiscoveryZookeeperIpFinder.java
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.fasterxml.jackson.annotation.JsonRootName;
import com.google.common.collect.Sets;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
@@ -43,7 +44,6 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.resources.LoggerResource;
import org.apache.ignite.spi.IgniteSpiException;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter;
-import org.codehaus.jackson.map.annotate.JsonRootName;
/**
* This TCP Discovery IP Finder uses Apache ZooKeeper (ZK) to locate peer nodes when bootstrapping in order to join
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd54a7/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index 78c16a9..48083b8 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -74,8 +74,7 @@
<hamcrest.version>1.2</hamcrest.version>
<httpclient.version>4.5.1</httpclient.version>
<httpcore.version>4.4.3</httpcore.version>
- <jackson.version>1.9.13</jackson.version>
- <jackson2.version>2.9.6</jackson2.version>
+ <jackson.version>2.9.6</jackson.version>
<javassist.version>3.20.0-GA</javassist.version>
<javax.cache.bundle.version>1.0.0_1</javax.cache.bundle.version>
<javax.cache.tck.version>1.0.1</javax.cache.tck.version>
[42/50] [abbrv] ignite git commit: IGNITE-9129 P2P class deployment
with ZK discovery fixed.
Posted by av...@apache.org.
IGNITE-9129 P2P class deployment with ZK discovery fixed.
Signed-off-by: agura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0e9bb1c0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0e9bb1c0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0e9bb1c0
Branch: refs/heads/ignite-8446
Commit: 0e9bb1c0e50ed8a29ccd3d6a9e2bd5311b6b7912
Parents: 0b39405
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Thu Aug 2 16:18:44 2018 +0300
Committer: agura <ag...@apache.org>
Committed: Thu Aug 2 16:19:24 2018 +0300
----------------------------------------------------------------------
.../continuous/GridContinuousProcessor.java | 72 +++++++++++---------
.../GridP2PContinuousDeploymentSelfTest.java | 70 ++++++++++++++++++-
.../zk/ZookeeperDiscoverySpiTestSuite2.java | 3 +
3 files changed, 109 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0e9bb1c0/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 2d48b7d..6723ea4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -544,7 +544,7 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
* @param routineInfo Routine info.
*/
private void startDiscoveryDataRoutine(ContinuousRoutineInfo routineInfo) {
- IgnitePredicate<ClusterNode> nodeFilter = null;
+ IgnitePredicate<ClusterNode> nodeFilter;
try {
if (routineInfo.nodeFilter != null) {
@@ -552,6 +552,8 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
ctx.resource().injectGeneric(nodeFilter);
}
+ else
+ nodeFilter = null;
}
catch (IgniteCheckedException e) {
U.error(log, "Failed to unmarshal continuous routine filter, ignore routine [" +
@@ -561,45 +563,47 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
return;
}
- if (nodeFilter == null || nodeFilter.apply(ctx.discovery().localNode())) {
- GridContinuousHandler hnd;
+ ctx.discovery().localJoinFuture().listen(f -> ctx.closure().runLocalSafe(() -> {
+ if (nodeFilter == null || nodeFilter.apply(ctx.discovery().localNode())) {
+ GridContinuousHandler hnd;
- try {
- hnd = U.unmarshal(marsh, routineInfo.hnd, U.resolveClassLoader(ctx.config()));
+ try {
+ hnd = U.unmarshal(marsh, routineInfo.hnd, U.resolveClassLoader(ctx.config()));
- if (ctx.config().isPeerClassLoadingEnabled())
- hnd.p2pUnmarshal(routineInfo.srcNodeId, ctx);
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to unmarshal continuous routine handler, ignore routine [" +
- "routineId=" + routineInfo.routineId +
- ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
+ if (ctx.config().isPeerClassLoadingEnabled())
+ hnd.p2pUnmarshal(routineInfo.srcNodeId, ctx);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to unmarshal continuous routine handler, ignore routine [" +
+ "routineId=" + routineInfo.routineId +
+ ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
- return;
- }
+ return;
+ }
- try {
- registerHandler(routineInfo.srcNodeId,
- routineInfo.routineId,
- hnd,
- routineInfo.bufSize,
- routineInfo.interval,
- routineInfo.autoUnsubscribe,
- false);
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to register continuous routine handler, ignore routine [" +
- "routineId=" + routineInfo.routineId +
- ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
+ try {
+ registerHandler(routineInfo.srcNodeId,
+ routineInfo.routineId,
+ hnd,
+ routineInfo.bufSize,
+ routineInfo.interval,
+ routineInfo.autoUnsubscribe,
+ false);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to register continuous routine handler, ignore routine [" +
+ "routineId=" + routineInfo.routineId +
+ ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
+ }
}
- }
- else {
- if (log.isDebugEnabled()) {
- log.debug("Do not register continuous routine, rejected by node filter [" +
- "routineId=" + routineInfo.routineId +
- ", srcNodeId=" + routineInfo.srcNodeId + ']');
+ else {
+ if (log.isDebugEnabled()) {
+ log.debug("Do not register continuous routine, rejected by node filter [" +
+ "routineId=" + routineInfo.routineId +
+ ", srcNodeId=" + routineInfo.srcNodeId + ']');
+ }
}
- }
+ }));
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/0e9bb1c0/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PContinuousDeploymentSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PContinuousDeploymentSelfTest.java b/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PContinuousDeploymentSelfTest.java
index 4ea9037..28dab3f 100644
--- a/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PContinuousDeploymentSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PContinuousDeploymentSelfTest.java
@@ -17,9 +17,14 @@
package org.apache.ignite.p2p;
+import java.util.UUID;
import org.apache.ignite.Ignite;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -49,6 +54,12 @@ public class GridP2PContinuousDeploymentSelfTest extends GridCommonAbstractTest
/** Second test task name. */
private static final String TEST_TASK_2 = "org.apache.ignite.tests.p2p.GridP2PContinuousDeploymentTask2";
+ /** Test predicate. */
+ private static final String TEST_PREDICATE = "org.apache.ignite.tests.p2p.GridEventConsumeFilter";
+
+ /** Client mode. */
+ private boolean clientMode;
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -66,6 +77,11 @@ public class GridP2PContinuousDeploymentSelfTest extends GridCommonAbstractTest
cfg.setDiscoverySpi(disco);
+ cfg.setPeerClassLoadingEnabled(true);
+
+ if (clientMode)
+ cfg.setClientMode(true);
+
return cfg;
}
@@ -85,8 +101,13 @@ public class GridP2PContinuousDeploymentSelfTest extends GridCommonAbstractTest
}
/** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- startGridsMultiThreaded(GRID_CNT);
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
}
/**
@@ -94,6 +115,8 @@ public class GridP2PContinuousDeploymentSelfTest extends GridCommonAbstractTest
*/
@SuppressWarnings("unchecked")
public void testDeployment() throws Exception {
+ startGridsMultiThreaded(GRID_CNT);
+
Ignite ignite = startGrid(IGNITE_INSTANCE_NAME);
Class cls = getExternalClassLoader().loadClass(TEST_TASK_1);
@@ -110,4 +133,47 @@ public class GridP2PContinuousDeploymentSelfTest extends GridCommonAbstractTest
stopGrid(IGNITE_INSTANCE_NAME);
}
+
+ /**
+ * Tests that server node joins correctly to existing cluster if it has deployed user class with enabled P2P.
+ *
+ * @throws Exception If failed.
+ */
+ public void testServerJoinWithP2PClassDeployedInCluster() throws Exception {
+ startGrids(GRID_CNT);
+
+ ClassLoader extLdr = getExternalClassLoader();
+
+ clientMode = true;
+
+ Ignite client = startGrid(2);
+
+ Class<?> cls = extLdr.loadClass(TEST_PREDICATE);
+
+ client.events().remoteListen(
+ new IgniteBiPredicate<UUID, Event>() {
+ @Override public boolean apply(UUID uuid, Event event) {
+ return true;
+ }
+ },
+ (IgnitePredicate<Event>) cls.newInstance(),
+ EventType.EVT_CACHE_OBJECT_PUT
+ );
+
+ clientMode = false;
+
+ Ignite srv = startGrid(3);
+
+ srv.events().remoteListen(
+ new IgniteBiPredicate<UUID, Event>() {
+ @Override public boolean apply(UUID uuid, Event event) {
+ return true;
+ }
+ },
+ (IgnitePredicate<Event>) cls.newInstance(),
+ EventType.EVT_CACHE_OBJECT_PUT
+ );
+
+ awaitPartitionMapExchange();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/0e9bb1c0/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
index ddb003b..012366f 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
@@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.distributed.replicated.Ignite
import org.apache.ignite.internal.processors.cache.multijvm.GridCacheAtomicMultiJvmFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.multijvm.GridCachePartitionedMultiJvmFullApiSelfTest;
import org.apache.ignite.internal.processors.continuous.GridEventConsumeSelfTest;
+import org.apache.ignite.p2p.GridP2PContinuousDeploymentSelfTest;
import org.apache.ignite.util.GridCommandHandlerTest;
/**
@@ -91,6 +92,8 @@ public class ZookeeperDiscoverySpiTestSuite2 extends ZookeeperDiscoverySpiAbstra
suite.addTestSuite(GridCommandHandlerTest.class);
+ suite.addTestSuite(GridP2PContinuousDeploymentSelfTest.class);
+
return suite;
}
}
[18/50] [abbrv] ignite git commit: IGNITE-9034: [ML] Add Estimator
API support to TensorFlow cluster on top of Apache Ignite.
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowUserScriptRunner.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowUserScriptRunner.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowUserScriptRunner.java
new file mode 100644
index 0000000..3dcd5f8
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowUserScriptRunner.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.cluster.util;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Map;
+import java.util.StringJoiner;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import org.apache.commons.io.IOUtils;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.tensorflow.cluster.TensorFlowJobArchive;
+import org.apache.ignite.tensorflow.cluster.spec.TensorFlowClusterSpec;
+import org.apache.ignite.tensorflow.cluster.spec.TensorFlowServerAddressSpec;
+import org.apache.ignite.tensorflow.core.util.AsyncNativeProcessRunner;
+import org.apache.ignite.tensorflow.core.util.NativeProcessRunner;
+
+/**
+ * Utils class that helps to start and stop user script process.
+ */
+public class TensorFlowUserScriptRunner extends AsyncNativeProcessRunner {
+ /** Ignite logger. */
+ private final IgniteLogger log;
+
+ /** Job archive that will be extracted and used as working directory for the native process. */
+ private final TensorFlowJobArchive jobArchive;
+
+ /** TensorFlow cluster specification. */
+ private final TensorFlowClusterSpec clusterSpec;
+
+ /** Output stream data consumer. */
+ private final Consumer<String> out;
+
+ /** Error stream data consumer. */
+ private final Consumer<String> err;
+
+ /** Working directory of the user script process. */
+ private File workingDir;
+
+ /**
+ * Constructs a new instance of TensorFlow user script runner.
+ *
+ * @param ignite Ignite instance.
+ * @param executor Executor to be used in {@link AsyncNativeProcessRunner}.
+ * @param jobArchive Job archive that will be extracted and used as working directory for the native process.
+ * @param clusterSpec TensorFlow cluster specification.
+ * @param out Output stream data consumer.
+ * @param err Error stream data consumer.
+ */
+ public TensorFlowUserScriptRunner(Ignite ignite, ExecutorService executor, TensorFlowJobArchive jobArchive,
+ TensorFlowClusterSpec clusterSpec, Consumer<String> out, Consumer<String> err) {
+ super(ignite, executor);
+
+ this.log = ignite.log().getLogger(TensorFlowUserScriptRunner.class);
+
+ this.jobArchive = jobArchive;
+ this.clusterSpec = clusterSpec;
+ this.out = out;
+ this.err = err;
+ }
+
+ /** {@inheritDoc} */
+ @Override public NativeProcessRunner doBefore() {
+ try {
+ workingDir = Files.createTempDirectory("tf_us_").toFile();
+ log.debug("Directory has been created [path=" + workingDir.getAbsolutePath() + "]");
+
+ unzip(jobArchive.getData(), workingDir);
+ log.debug("Job archive has been extracted [path=" + workingDir.getAbsolutePath() + "]");
+
+ return prepareNativeProcessRunner();
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void doAfter() {
+ if (workingDir != null) {
+ delete(workingDir);
+ log.debug("Directory has been deleted [path=" + workingDir.getAbsolutePath() + "]");
+ }
+ }
+
+ /**
+ * Prepares process builder and specifies working directory and command to be run.
+ *
+ * @return Prepared process builder.
+ */
+ private NativeProcessRunner prepareNativeProcessRunner() {
+ if (workingDir == null)
+ throw new IllegalStateException("Working directory is not created");
+
+ ProcessBuilder procBuilder = new ProcessBuilder();
+
+ procBuilder.directory(workingDir);
+ procBuilder.command(jobArchive.getCommands());
+
+ Map<String, String> env = procBuilder.environment();
+ env.put("PYTHONPATH", workingDir.getAbsolutePath());
+ env.put("TF_CONFIG", formatTfConfigVar());
+ env.put("TF_WORKERS", formatTfWorkersVar());
+ env.put("TF_CHIEF_SERVER", formatTfChiefServerVar());
+
+ return new NativeProcessRunner(procBuilder, null, out, err);
+ }
+
+ /**
+ * Formats "TF_CONFIG" variable to be passed into user script.
+ *
+ * @return Formatted "TF_CONFIG" variable to be passed into user script.
+ */
+ private String formatTfConfigVar() {
+ return new StringBuilder()
+ .append("{\"cluster\" : ")
+ .append(clusterSpec.format(Ignition.ignite()))
+ .append(", ")
+ .append("\"task\": {\"type\" : \"" + TensorFlowClusterResolver.CHIEF_JOB_NAME + "\", \"index\": 0}}")
+ .toString();
+ }
+
+ /**
+ * Formats "TF_WORKERS" variable to be passed into user script.
+ *
+ * @return Formatted "TF_WORKERS" variable to be passed into user script.
+ */
+ private String formatTfWorkersVar() {
+ StringJoiner joiner = new StringJoiner(", ");
+
+ int cnt = clusterSpec.getJobs().get(TensorFlowClusterResolver.WORKER_JOB_NAME).size();
+ for (int i = 0; i < cnt; i++)
+ joiner.add("\"/job:" + TensorFlowClusterResolver.WORKER_JOB_NAME + "/task:" + i + "\"");
+
+ return "[" + joiner + "]";
+ }
+
+ /**
+ * Formats "TF_CHIEF_SERVER" variable to be passed into user script.
+ *
+ * @return Formatted "TF_CHIEF_SERVER" variable to be passed into user script.
+ */
+ private String formatTfChiefServerVar() {
+ List<TensorFlowServerAddressSpec> tasks = clusterSpec.getJobs().get(TensorFlowClusterResolver.CHIEF_JOB_NAME);
+
+ if (tasks == null || tasks.size() != 1)
+ throw new IllegalStateException("TensorFlow cluster specification should contain exactly one chief task");
+
+ TensorFlowServerAddressSpec addrSpec = tasks.iterator().next();
+
+ return "grpc://" + addrSpec.format(Ignition.ignite());
+ }
+
+ /**
+ * Clears given file or directory recursively.
+ *
+ * @param file File or directory to be cleaned,
+ */
+ private void delete(File file) {
+ if (file.isDirectory()) {
+ String[] files = file.list();
+
+ if (files != null && files.length != 0)
+ for (String fileToBeDeleted : files)
+ delete(new File(file, fileToBeDeleted));
+
+ if (!file.delete())
+ throw new IllegalStateException("Can't delete directory [path=" + file.getAbsolutePath() + "]");
+ }
+ else {
+ if (!file.delete())
+ throw new IllegalStateException("Can't delete file [path=" + file.getAbsolutePath() + "]");
+ }
+ }
+
+ /**
+ * Extracts specified zip archive into specified directory.
+ *
+ * @param data Zip archive to be extracted.
+ * @param extractTo Target directory.
+ */
+ private void unzip(byte[] data, File extractTo) {
+ try (ZipInputStream zipStream = new ZipInputStream(new ByteArrayInputStream(data))) {
+ ZipEntry entry;
+ while ((entry = zipStream.getNextEntry()) != null) {
+ File file = new File(extractTo, entry.getName());
+
+ if (entry.isDirectory() && !file.exists()) {
+ boolean created = file.mkdirs();
+ if (!created)
+ throw new IllegalStateException("Can't create directory [path=" + file.getAbsolutePath() + "]");
+ }
+ else {
+ if (!file.getParentFile().exists()) {
+ boolean created = file.getParentFile().mkdirs();
+ if (!created)
+ throw new IllegalStateException("Can't create directory [path=" +
+ file.getParentFile().getAbsolutePath() + "]");
+ }
+
+ try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(file))) {
+ IOUtils.copy(zipStream, out);
+ }
+ }
+ }
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManager.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManager.java
index 0ef81bc..c825448 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManager.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManager.java
@@ -17,18 +17,17 @@
package org.apache.ignite.tensorflow.core;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
-import java.io.Serializable;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
/**
* Process manager that allows to run and maintain processes in the cluster.
*
* @param <R> Type of task to be run.
*/
-public interface ProcessManager<R> extends Serializable {
+public interface ProcessManager<R> {
/**
* Starts the processes by the given specifications.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManagerWrapper.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManagerWrapper.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManagerWrapper.java
index b66b54f..4f10e83 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManagerWrapper.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/ProcessManagerWrapper.java
@@ -17,11 +17,11 @@
package org.apache.ignite.tensorflow.core;
-import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProcessStatus;
/**
* Process manager wrapper that allows to define how one type of process specification should be transformed into
@@ -31,9 +31,6 @@ import java.util.UUID;
* @param <R> Type of accepted process specifications.
*/
public abstract class ProcessManagerWrapper<T, R> implements ProcessManager<R> {
- /** */
- private static final long serialVersionUID = -6397225095261457524L;
-
/** Delegate. */
private final ProcessManager<T> delegate;
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManager.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManager.java
index 027ece3..a25ff97 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManager.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManager.java
@@ -17,7 +17,6 @@
package org.apache.ignite.tensorflow.core.longrunning;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -25,7 +24,6 @@ import java.util.Map;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.function.Function;
-import java.util.function.Supplier;
import org.apache.ignite.Ignite;
import org.apache.ignite.cluster.ClusterGroup;
import org.apache.ignite.cluster.ClusterGroupEmptyException;
@@ -41,22 +39,18 @@ import org.apache.ignite.tensorflow.core.longrunning.task.util.LongRunningProces
* Long running process manager that allows to start, stop and make other actions with long running processes.
*/
public class LongRunningProcessManager implements ProcessManager<LongRunningProcess> {
- /** */
- private static final long serialVersionUID = 1151455641358063287L;
-
- /** Ignite instance supplier. */
- private final Supplier<Ignite> igniteSupplier;
+ /** Ignite instance. */
+ private final Ignite ignite;
/**
* Constructs a new instance of long running process manager.
*
- * @param igniteSupplier Ignite instance supplier.
- * @param <T> Type of serializable supplier.
+ * @param ignite Ignite instance.
*/
- public <T extends Supplier<Ignite> & Serializable> LongRunningProcessManager(T igniteSupplier) {
- assert igniteSupplier != null : "Ignite supplier should not be null";
+ public LongRunningProcessManager(Ignite ignite) {
+ assert ignite != null : "Ignite instance should not be null";
- this.igniteSupplier = igniteSupplier;
+ this.ignite = ignite;
}
/** {@inheritDoc} */
@@ -100,7 +94,6 @@ public class LongRunningProcessManager implements ProcessManager<LongRunningProc
List<T> nodeProcesses = params.get(nodeId);
LongRunningProcessTask<List<E>> task = taskSupplier.apply(nodeProcesses);
- Ignite ignite = igniteSupplier.get();
ClusterGroup clusterGrp = ignite.cluster().forNodeId(nodeId);
try {
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStartTask.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStartTask.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStartTask.java
index 1d08519..04f90d3 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStartTask.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/longrunning/task/LongRunningProcessStartTask.java
@@ -17,13 +17,13 @@
package org.apache.ignite.tensorflow.core.longrunning.task;
-import org.apache.ignite.tensorflow.core.longrunning.LongRunningProcess;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import org.apache.ignite.tensorflow.core.longrunning.LongRunningProcess;
import org.apache.ignite.tensorflow.core.util.CustomizableThreadFactory;
/**
@@ -78,7 +78,7 @@ public class LongRunningProcessStartTask extends LongRunningProcessTask<List<UUI
*/
private Future<?> runTask(Runnable task) {
return Executors
- .newSingleThreadExecutor(new CustomizableThreadFactory("LONG_RUNNING_PROCESS_TASK", true))
+ .newSingleThreadExecutor(new CustomizableThreadFactory("tf-long-running", true))
.submit(task);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcess.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcess.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcess.java
index df36ba9..2ad3c9d 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcess.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcess.java
@@ -20,6 +20,7 @@ package org.apache.ignite.tensorflow.core.nativerunning;
import java.io.Serializable;
import java.util.UUID;
import java.util.function.Supplier;
+import org.apache.ignite.tensorflow.util.SerializableSupplier;
/**
* Native process specification.
@@ -29,7 +30,7 @@ public class NativeProcess implements Serializable {
private static final long serialVersionUID = -7056800139746134956L;
/** Process builder supplier. */
- private final Supplier<ProcessBuilder> procBuilderSupplier;
+ private final SerializableSupplier<ProcessBuilder> procBuilderSupplier;
/** Stdin of the process. */
private final String stdin;
@@ -44,8 +45,7 @@ public class NativeProcess implements Serializable {
* @param stdin Stdin of the process.
* @param nodeId Node identifier.
*/
- public <T extends Supplier<ProcessBuilder> & Serializable> NativeProcess(T procBuilderSupplier, String stdin,
- UUID nodeId) {
+ public NativeProcess(SerializableSupplier<ProcessBuilder> procBuilderSupplier, String stdin, UUID nodeId) {
assert procBuilderSupplier != null : "Process builder supplier should not be null";
assert nodeId != null : "Node identifier should not be null";
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcessManager.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcessManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcessManager.java
index 60cd89b..5accf3d 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcessManager.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/NativeProcessManager.java
@@ -17,8 +17,6 @@
package org.apache.ignite.tensorflow.core.nativerunning;
-import java.io.Serializable;
-import java.util.function.Supplier;
import org.apache.ignite.Ignite;
import org.apache.ignite.tensorflow.core.ProcessManager;
import org.apache.ignite.tensorflow.core.ProcessManagerWrapper;
@@ -30,17 +28,13 @@ import org.apache.ignite.tensorflow.core.nativerunning.task.NativeProcessStartTa
* Native process manager that allows to start, stop and make other actions with native processes.
*/
public class NativeProcessManager extends ProcessManagerWrapper<LongRunningProcess, NativeProcess> {
- /** */
- private static final long serialVersionUID = 718119807915504045L;
-
/**
* Constructs a new native process manager.
*
- * @param igniteSupplier Ignite instance supplier.
- * @param <T> Type of serializable supplier.
+ * @param ignite Ignite instance.
*/
- public <T extends Supplier<Ignite> & Serializable> NativeProcessManager(T igniteSupplier) {
- super(new LongRunningProcessManager(igniteSupplier));
+ public NativeProcessManager(Ignite ignite) {
+ super(new LongRunningProcessManager(ignite));
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/task/NativeProcessStartTask.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/task/NativeProcessStartTask.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/task/NativeProcessStartTask.java
index 8fc28a5..ae9e2b9 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/task/NativeProcessStartTask.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/nativerunning/task/NativeProcessStartTask.java
@@ -18,16 +18,11 @@
package org.apache.ignite.tensorflow.core.nativerunning.task;
import java.util.function.Supplier;
-import org.apache.ignite.tensorflow.core.nativerunning.NativeProcess;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintStream;
-import java.io.PrintWriter;
-import java.util.Scanner;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.Ignition;
import org.apache.ignite.lang.IgniteRunnable;
-import org.apache.ignite.tensorflow.core.util.CustomizableThreadFactory;
+import org.apache.ignite.tensorflow.core.nativerunning.NativeProcess;
+import org.apache.ignite.tensorflow.core.util.NativeProcessRunner;
/**
* Task that starts native process by its specification.
@@ -55,62 +50,26 @@ public class NativeProcessStartTask implements IgniteRunnable {
Supplier<ProcessBuilder> procBuilderSupplier = procSpec.getProcBuilderSupplier();
ProcessBuilder procBuilder = procBuilderSupplier.get();
- Process proc;
- try {
- proc = procBuilder.start();
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- Thread shutdownHook = new Thread(proc::destroy);
- Runtime.getRuntime().addShutdownHook(shutdownHook);
+ NativeProcessRunner procRunner = new NativeProcessRunner(
+ procBuilder,
+ procSpec.getStdin(),
+ System.out::println,
+ System.err::println
+ );
- Future<?> outForward = forwardStream(proc.getInputStream(), System.out);
- Future<?> errForward = forwardStream(proc.getErrorStream(), System.err);
+ IgniteLogger log = Ignition.ignite().log().getLogger(NativeProcessStartTask.class);
try {
- if (procSpec.getStdin() != null) {
- PrintWriter writer = new PrintWriter(proc.getOutputStream());
- writer.println(procSpec.getStdin());
- writer.flush();
- }
-
- int status;
- try {
- status = proc.waitFor();
- }
- catch (InterruptedException e) {
- proc.destroy();
- status = proc.exitValue();
- }
-
- Runtime.getRuntime().removeShutdownHook(shutdownHook);
-
- if (status != 0)
- throw new IllegalStateException("Native process exit status is " + status);
+ log.debug("Starting native process");
+ procRunner.startAndWait();
+ log.debug("Native process completed");
}
- finally {
- outForward.cancel(true);
- errForward.cancel(true);
+ catch (InterruptedException e) {
+ log.debug("Native process interrupted");
+ }
+ catch (Exception e) {
+ log.error("Native process failed", e);
+ throw e;
}
- }
-
- /**
- * Forwards stream.
- *
- * @param src Source stream.
- * @param dst Destination stream.
- * @return Future that allows to interrupt forwarding.
- */
- private Future<?> forwardStream(InputStream src, PrintStream dst) {
- return Executors
- .newSingleThreadExecutor(new CustomizableThreadFactory("NATIVE_PROCESS_FORWARD_STREAM", true))
- .submit(() -> {
- Scanner scanner = new Scanner(src);
-
- while (!Thread.currentThread().isInterrupted() && scanner.hasNextLine())
- dst.println(scanner.nextLine());
- });
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/pythonrunning/PythonProcessBuilderSupplier.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/pythonrunning/PythonProcessBuilderSupplier.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/pythonrunning/PythonProcessBuilderSupplier.java
new file mode 100644
index 0000000..e59ab00
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/pythonrunning/PythonProcessBuilderSupplier.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.core.pythonrunning;
+
+import org.apache.ignite.tensorflow.util.SerializableSupplier;
+
+/**
+ * Python process builder supplier that is used to create Python process builder.
+ */
+public class PythonProcessBuilderSupplier implements SerializableSupplier<ProcessBuilder> {
+ /** */
+ private static final long serialVersionUID = 7181937306294456125L;
+
+ /** Python environment variable name. */
+ private static final String PYTHON_ENV_NAME = "PYTHON";
+
+ /** Interactive flag (allows to used standard input to pass Python script). */
+ private final boolean interactive;
+
+ /**
+ * Constructs a new instance of Python process builder supplier.
+ *
+ * @param interactive Interactive flag (allows to used standard input to pass Python script).
+ */
+ public PythonProcessBuilderSupplier(boolean interactive) {
+ this.interactive = interactive;
+ }
+
+ /**
+ * Returns process builder to be used to start Python process.
+ *
+ * @return Process builder to be used to start Python process.
+ */
+ public ProcessBuilder get() {
+ String python = System.getenv(PYTHON_ENV_NAME);
+
+ if (python == null)
+ python = "python3";
+
+ return interactive ? new ProcessBuilder(python, "-i") : new ProcessBuilder(python);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/pythonrunning/PythonProcessManager.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/pythonrunning/PythonProcessManager.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/pythonrunning/PythonProcessManager.java
index de35ff9..1f6c11e 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/pythonrunning/PythonProcessManager.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/pythonrunning/PythonProcessManager.java
@@ -17,8 +17,6 @@
package org.apache.ignite.tensorflow.core.pythonrunning;
-import java.io.Serializable;
-import java.util.function.Supplier;
import org.apache.ignite.Ignite;
import org.apache.ignite.tensorflow.core.ProcessManager;
import org.apache.ignite.tensorflow.core.ProcessManagerWrapper;
@@ -29,17 +27,13 @@ import org.apache.ignite.tensorflow.core.nativerunning.NativeProcessManager;
* Python process manager that allows to start, stop and make other actions with python processes.
*/
public class PythonProcessManager extends ProcessManagerWrapper<NativeProcess, PythonProcess> {
- /** */
- private static final long serialVersionUID = -7095409565854538504L;
-
/**
* Constructs a new instance of python process manager.
*
- * @param igniteSupplier Ignite instance supplier.
- * @param <T> Type of serializable supplier.
+ * @param ignite Ignite instance.
*/
- public <T extends Supplier<Ignite> & Serializable> PythonProcessManager(T igniteSupplier) {
- this(new NativeProcessManager(igniteSupplier));
+ public PythonProcessManager(Ignite ignite) {
+ this(new NativeProcessManager(ignite));
}
/**
@@ -54,30 +48,9 @@ public class PythonProcessManager extends ProcessManagerWrapper<NativeProcess, P
/** {@inheritDoc} */
@Override protected NativeProcess transformSpecification(PythonProcess spec) {
return new NativeProcess(
- new PythonProcessBuilderSupplier(),
+ new PythonProcessBuilderSupplier(true),
spec.getStdin(),
spec.getNodeId()
);
}
-
- /**
- * Python process builder supplier that is used to create Python process builder.
- */
- private static class PythonProcessBuilderSupplier implements Supplier<ProcessBuilder>, Serializable {
- /** */
- private static final long serialVersionUID = 8497087649461965914L;
-
- /** Python environment variable name. */
- private static final String PYTHON_ENV_NAME = "PYTHON";
-
- /** {@inheritDoc} */
- @Override public ProcessBuilder get() {
- String python = System.getenv(PYTHON_ENV_NAME);
-
- if (python == null)
- python = "python3";
-
- return new ProcessBuilder(python, "-i");
- }
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/AsyncNativeProcessRunner.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/AsyncNativeProcessRunner.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/AsyncNativeProcessRunner.java
new file mode 100644
index 0000000..b336b97
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/AsyncNativeProcessRunner.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.core.util;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteLogger;
+
+/**
+ * Asynchronous native process runner.
+ */
+public abstract class AsyncNativeProcessRunner {
+ /** Ignite logger. */
+ private final IgniteLogger log;
+
+ /** Executors that is used to start async native process. */
+ private final ExecutorService executor;
+
+ /** Future of the async process process. */
+ private Future<?> fut;
+
+ /**
+ * Constructs a new asynchronous native process runner.
+ *
+ * @param ignite Ignite instance.
+ * @param executor Executor.
+ */
+ public AsyncNativeProcessRunner(Ignite ignite, ExecutorService executor) {
+ this.log = ignite.log().getLogger(AsyncNativeProcessRunner.class);
+ this.executor = executor;
+ }
+
+ /**
+ * Method that should be called before starting the process.
+ *
+ * @return Prepared native process runner.
+ */
+ public abstract NativeProcessRunner doBefore();
+
+ /**
+ * Method that should be called after starting the process.
+ */
+ public abstract void doAfter();
+
+ /**
+ * Starts the process in separate thread.
+ */
+ public synchronized void start() {
+ if (fut != null)
+ throw new IllegalStateException("Async native process has already been started");
+
+ NativeProcessRunner procRunner = doBefore();
+
+ fut = executor.submit(() -> {
+ while (!Thread.currentThread().isInterrupted()) {
+ try {
+ log.debug("Starting native process");
+ procRunner.startAndWait();
+ log.debug("Native process completed");
+ break;
+ }
+ catch (InterruptedException e) {
+ log.debug("Native process interrupted");
+ break;
+ }
+ catch (Exception e) {
+ log.error("Native process failed", e);
+ }
+ }
+
+ doAfter();
+ });
+ }
+
+ /**
+ * Stops the process.
+ */
+ public synchronized void stop() {
+ if (fut != null && !fut.isDone())
+ fut.cancel(true);
+ }
+
+ /**
+ * Checks if process is already completed.
+ *
+ * @return {@code true} if process completed, otherwise {@code false}.
+ */
+ public boolean isCompleted() {
+ return fut != null && fut.isDone();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/NativeProcessRunner.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/NativeProcessRunner.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/NativeProcessRunner.java
new file mode 100644
index 0000000..38af26d
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/core/util/NativeProcessRunner.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.core.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.util.Scanner;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+
+/**
+ * Utils class that helps to start native processes.
+ */
+public class NativeProcessRunner {
+ /** Thread name to be used by threads that forward streams. */
+ private static final String NATIVE_PROCESS_FORWARD_STREAM_THREAD_NAME = "tf-forward-native-output";
+
+ /** Process builder. */
+ private final ProcessBuilder procBuilder;
+
+ /** Standard input of the process. */
+ private final String stdin;
+
+ /** Output stream data consumer. */
+ private final Consumer<String> out;
+
+ /** Error stream data consumer. */
+ private final Consumer<String> err;
+
+ /**
+ * Constructs a new instance of native process runner.
+ *
+ * @param procBuilder Process builder.
+ * @param stdin Standard input of the process.
+ * @param out Output stream data consumer.
+ * @param err Error stream data consumer.
+ */
+ public NativeProcessRunner(ProcessBuilder procBuilder, String stdin, Consumer<String> out, Consumer<String> err) {
+ this.procBuilder = procBuilder;
+ this.stdin = stdin;
+ this.out = out;
+ this.err = err;
+ }
+
+ /**
+ * Starts the native process and waits it to be completed successfully or with exception.
+ */
+ public void startAndWait() throws InterruptedException {
+ Process proc;
+ try {
+ proc = procBuilder.start();
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ AtomicBoolean shutdown = new AtomicBoolean();
+
+ Thread shutdownHook = new Thread(() -> {
+ shutdown.set(true);
+ proc.destroy();
+ });
+
+ Runtime.getRuntime().addShutdownHook(shutdownHook);
+
+ Future<?> outForward = forwardStream(proc.getInputStream(), out);
+ Future<?> errForward = forwardStream(proc.getErrorStream(), err);
+
+ try {
+ if (stdin != null) {
+ PrintWriter writer = new PrintWriter(proc.getOutputStream());
+ writer.println(stdin);
+ writer.flush();
+ }
+
+ int status;
+ try {
+ status = proc.waitFor();
+ }
+ catch (InterruptedException e) {
+ proc.destroy();
+ throw e;
+ }
+
+ if (!shutdown.get()) {
+ Runtime.getRuntime().removeShutdownHook(shutdownHook);
+
+ if (status != 0)
+ throw new IllegalStateException("Native process exit [status=" + status + "]");
+ }
+ }
+ finally {
+ outForward.cancel(true);
+ errForward.cancel(true);
+ }
+ }
+
+ /**
+ * Forwards stream.
+ *
+ * @param src Source stream.
+ * @param dst Destination stream.
+ * @return Future that allows to interrupt forwarding.
+ */
+ private Future<?> forwardStream(InputStream src, Consumer<String> dst) {
+ return Executors
+ .newSingleThreadExecutor(new CustomizableThreadFactory(NATIVE_PROCESS_FORWARD_STREAM_THREAD_NAME, true))
+ .submit(() -> {
+ Scanner scanner = new Scanner(src);
+
+ while (!Thread.currentThread().isInterrupted() && scanner.hasNextLine())
+ dst.accept(scanner.nextLine());
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/JobSubmitter.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/JobSubmitter.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/JobSubmitter.java
new file mode 100644
index 0000000..0a7cae6
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/JobSubmitter.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.submitter;
+
+import org.apache.ignite.tensorflow.submitter.command.RootCommand;
+import picocli.CommandLine;
+
+/**
+ * Main class of the job submitter application that allows to submit TensorFlow jobs to be run within Ignite cluster.
+ */
+public class JobSubmitter {
+ /**
+ * Main method.
+ *
+ * @param args Arguments.
+ */
+ public static void main(String... args) {
+ CommandLine.run(new RootCommand(), System.out, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AbstractCommand.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AbstractCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AbstractCommand.java
new file mode 100644
index 0000000..4d2fc18
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AbstractCommand.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.submitter.command;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.logger.slf4j.Slf4jLogger;
+import picocli.CommandLine;
+
+/**
+ * Abstract command that contains options common for all commands.
+ */
+public abstract class AbstractCommand implements Runnable {
+ /** Ignite node configuration path. */
+ @CommandLine.Option(names = { "-c", "--config" }, description = "Apache Ignite client configuration.")
+ protected String cfg;
+
+ /**
+ * Returns Ignite instance based on configuration specified in {@link #cfg} field.
+ *
+ * @return Ignite instance.
+ */
+ protected Ignite getIgnite() {
+ if (cfg != null)
+ return Ignition.start(cfg);
+ else {
+ IgniteConfiguration igniteCfg = new IgniteConfiguration();
+ igniteCfg.setGridLogger(new Slf4jLogger());
+ igniteCfg.setClientMode(true);
+
+ return Ignition.start(igniteCfg);
+ }
+ }
+
+ /** */
+ public void setCfg(String cfg) {
+ this.cfg = cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AttachCommand.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AttachCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AttachCommand.java
new file mode 100644
index 0000000..946aa08
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/AttachCommand.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.submitter.command;
+
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.tensorflow.cluster.TensorFlowClusterGatewayManager;
+import picocli.CommandLine;
+
+/**
+ * Command "attach" that is used to attach to running TensorFlow cluster and receive output of the user script.
+ */
+@CommandLine.Command(
+ name = "attach",
+ description = "Attaches to running TensorFlow cluster (user script process).",
+ mixinStandardHelpOptions = true
+)
+public class AttachCommand extends AbstractCommand {
+ /** TensorFlow cluster identifier. */
+ @CommandLine.Parameters(paramLabel = "CLUSTER_ID", description = "Cluster identifier.")
+ private UUID clusterId;
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try (Ignite ignite = getIgnite()) {
+ TensorFlowClusterGatewayManager mgr = new TensorFlowClusterGatewayManager(ignite);
+
+ mgr.listenToClusterUserScript(clusterId, System.out::println, System.err::println);
+ }
+ }
+
+ /** */
+ public void setClusterId(UUID clusterId) {
+ this.clusterId = clusterId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/PsCommand.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/PsCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/PsCommand.java
new file mode 100644
index 0000000..0538496
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/PsCommand.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.submitter.command;
+
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.tensorflow.cluster.TensorFlowCluster;
+import org.apache.ignite.tensorflow.cluster.TensorFlowClusterManager;
+import picocli.CommandLine;
+
+/**
+ * Command "ps" that is used to print identifiers of all running TensorFlow clusters.
+ */
+@CommandLine.Command(
+ name = "ps",
+ description = "Prints identifiers of all running TensorFlow clusters.",
+ mixinStandardHelpOptions = true
+)
+public class PsCommand extends AbstractCommand {
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try (Ignite ignite = getIgnite()) {
+ TensorFlowClusterManager mgr = new TensorFlowClusterManager(ignite);
+
+ Map<UUID, TensorFlowCluster> clusters = mgr.getAllClusters();
+
+ for (UUID clusterId : clusters.keySet())
+ System.out.println(clusterId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/RootCommand.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/RootCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/RootCommand.java
new file mode 100644
index 0000000..508ea7b
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/RootCommand.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.submitter.command;
+
+import picocli.CommandLine;
+
+/**
+ * Root command that aggregates all sub commands.
+ */
+@CommandLine.Command(
+ name = "ignite-tf",
+ description = "Apache Ignite and TensorFlow integration command line tool that allows to start, maintain and" +
+ " stop distributed deep learning utilizing Apache Ignite infrastructure and data.",
+ subcommands = {
+ StartCommand.class,
+ StopCommand.class,
+ AttachCommand.class,
+ PsCommand.class
+ },
+ mixinStandardHelpOptions = true
+)
+public class RootCommand extends AbstractCommand {
+ /** {@inheritDoc} */
+ @Override public void run() {
+ CommandLine.usage(this, System.out);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StartCommand.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StartCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StartCommand.java
new file mode 100644
index 0000000..082b363
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StartCommand.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.submitter.command;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.UUID;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+import org.apache.commons.io.IOUtils;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.tensorflow.cluster.TensorFlowClusterGatewayManager;
+import org.apache.ignite.tensorflow.cluster.TensorFlowJobArchive;
+import picocli.CommandLine;
+
+/**
+ * Command "start" that is used to start a new TensorFlow cluster on top of Apache Ignite.
+ */
+@CommandLine.Command(
+ name = "start",
+ description = "Starts a new TensorFlow cluster and attaches to user script process.",
+ mixinStandardHelpOptions = true
+)
+public class StartCommand extends AbstractCommand {
+ /** Upstream cache name. */
+ @CommandLine.Parameters(index = "0", paramLabel = "CACHE_NAME", description = "Upstream cache name.")
+ private String cacheName;
+
+ /** Job folder or archive. */
+ @CommandLine.Parameters(index = "1", paramLabel = "JOB_DIR", description = "Job folder (or zip archive).")
+ private String jobFolder;
+
+ /** Job command to be executed in cluster. */
+ @CommandLine.Parameters(index = "2", paramLabel = "JOB_CMD", description = "Job command.")
+ private String jobCmd;
+
+ /** Arguments of a job command to be executed in cluster. */
+ @CommandLine.Parameters(index = "3..*", paramLabel = "JOB_ARGS", description = "Job arguments.")
+ private String[] jobArguments;
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try (Ignite ignite = getIgnite()) {
+ UUID clusterId = UUID.randomUUID();
+ String[] commands = new String[jobArguments.length + 1];
+ commands[0] = jobCmd;
+ System.arraycopy(jobArguments, 0, commands, 1, commands.length - 1);
+
+ TensorFlowJobArchive jobArchive = new TensorFlowJobArchive(
+ cacheName,
+ zip(jobFolder),
+ commands
+ );
+
+ TensorFlowClusterGatewayManager mgr = new TensorFlowClusterGatewayManager(ignite);
+ mgr.createCluster(clusterId, jobArchive);
+
+ mgr.listenToClusterUserScript(clusterId, System.out::println, System.err::println);
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Archives specified folder or file into zip archive.
+ *
+ * @param jobArchivePath Path to folder to be archived.
+ * @return Byte array representing zip archive.
+ * @throws IOException In case of input/output exception.
+ */
+ private byte[] zip(String jobArchivePath) throws IOException {
+ Path path = Paths.get(jobArchivePath);
+ File file = path.toFile();
+
+ if (!file.exists())
+ throw new IllegalArgumentException("File doesn't exist [name=" + jobArchivePath + "]");
+
+ if (file.isDirectory())
+ return zipDirectory(file);
+ else if (jobArchivePath.endsWith(".zip"))
+ return zipArchive(file);
+ else
+ return zipFile(file);
+ }
+
+ /**
+ * Archives specified folder into zip archive.
+ *
+ * @param dir Directory to be archived.
+ * @return Byte array representing zip archive.
+ * @throws IOException In case of input/output exception.
+ */
+ private byte[] zipDirectory(File dir) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ try (ZipOutputStream zipFile = new ZipOutputStream(baos)) {
+ compressDirectoryToZip(dir.getAbsolutePath(), dir.getAbsolutePath(), zipFile);
+ }
+
+ return baos.toByteArray();
+ }
+
+ /**
+ * Archives specified file into zip archive.
+ *
+ * @param file File to be archived.
+ * @return Byte array representing zip archive.
+ * @throws IOException In case of input/output exception.
+ */
+ private byte[] zipFile(File file) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ try (ZipOutputStream zos = new ZipOutputStream(baos)) {
+ ZipEntry entry = new ZipEntry(file.getName());
+ zos.putNextEntry(entry);
+
+ try (FileInputStream in = new FileInputStream(file.getAbsolutePath())) {
+ IOUtils.copy(in, zos);
+ }
+ }
+
+ return baos.toByteArray();
+ }
+
+ /**
+ * Reads zip archive into byte array and returns this array.
+ *
+ * @param file Archive to be read.
+ * @return Byte array representing zip archive.
+ * @throws IOException In case of input/output exception.
+ */
+ private byte[] zipArchive(File file) throws IOException {
+ try (FileInputStream fis = new FileInputStream(file)) {
+ return IOUtils.toByteArray(fis);
+ }
+ }
+
+ /**
+ * Archives specified folder into zip output stream.
+ *
+ * @param rootDir Root directory.
+ * @param srcDir Source directory.
+ * @param out Zip output stream.
+ * @throws IOException In case of input/output exception.
+ */
+ private void compressDirectoryToZip(String rootDir, String srcDir, ZipOutputStream out) throws IOException {
+ File[] files = new File(srcDir).listFiles();
+
+ if (files != null) {
+ for (File file : files) {
+ if (file.isDirectory())
+ compressDirectoryToZip(rootDir, srcDir + File.separator + file.getName(), out);
+ else {
+ ZipEntry entry = new ZipEntry(srcDir.replace(rootDir, "")
+ + File.separator + file.getName());
+ out.putNextEntry(entry);
+
+ try (FileInputStream in = new FileInputStream(srcDir + File.separator + file.getName())) {
+ IOUtils.copy(in, out);
+ }
+ }
+ }
+ }
+ }
+
+ /** */
+ public void setCacheName(String cacheName) {
+ this.cacheName = cacheName;
+ }
+
+ /** */
+ public void setJobFolder(String jobFolder) {
+ this.jobFolder = jobFolder;
+ }
+
+ /** */
+ public void setJobCmd(String jobCmd) {
+ this.jobCmd = jobCmd;
+ }
+
+ /** */
+ public void setJobArguments(String[] jobArguments) {
+ this.jobArguments = jobArguments;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StopCommand.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StopCommand.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StopCommand.java
new file mode 100644
index 0000000..8890370
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/StopCommand.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.submitter.command;
+
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.tensorflow.cluster.TensorFlowClusterGatewayManager;
+import picocli.CommandLine;
+
+/**
+ * Command "stop" that is used to stop TensorFlow cluster.
+ */
+@CommandLine.Command(
+ name = "stop",
+ description = "Stops a running TensorFlow cluster.",
+ mixinStandardHelpOptions = true
+)
+public class StopCommand extends AbstractCommand {
+ /** Cluster identifier. */
+ @CommandLine.Parameters(paramLabel = "CLUSTER_ID", description = "Cluster identifier.")
+ private UUID clusterId;
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try (Ignite ignite = getIgnite()) {
+ TensorFlowClusterGatewayManager mgr = new TensorFlowClusterGatewayManager(ignite);
+ mgr.stopClusterIfExists(clusterId);
+ }
+ }
+
+ /** */
+ public void setClusterId(UUID clusterId) {
+ this.clusterId = clusterId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/package-info.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/package-info.java
new file mode 100644
index 0000000..7949feb
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/command/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * This package contains commands that command line tool provides. Pico CLI is used to make these commands maintainable.
+ */
+package org.apache.ignite.tensorflow.submitter.command;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/package-info.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/package-info.java
new file mode 100644
index 0000000..8288b16
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/submitter/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * This package contains classes that allow to use command line interface to submit jobs into TensorFlow in Apache
+ * Ignite infrastructure.
+ */
+package org.apache.ignite.tensorflow.submitter;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableConsumer.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableConsumer.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableConsumer.java
new file mode 100644
index 0000000..ece58aa
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableConsumer.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.util;
+
+import java.io.Serializable;
+import java.util.function.Consumer;
+
+/**
+ * Serializable consumer.
+ *
+ * @param <T> The type of the input to the operation.
+ */
+public interface SerializableConsumer<T> extends Consumer<T>, Serializable {
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableSupplier.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableSupplier.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableSupplier.java
new file mode 100644
index 0000000..768dbe1
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/SerializableSupplier.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tensorflow.util;
+
+import java.io.Serializable;
+import java.util.function.Supplier;
+
+/**
+ * Serializable supplier.
+ *
+ * @param <T> The type of results supplied by this supplier.
+ */
+public interface SerializableSupplier<T> extends Supplier<T>, Serializable {
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/package-info.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/package-info.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/package-info.java
new file mode 100644
index 0000000..8ed43c3
--- /dev/null
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/util/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Util classes used in {@link org.apache.ignite.tensorflow} package.
+ */
+package org.apache.ignite.tensorflow.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/sh/ignite-tf.sh
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/sh/ignite-tf.sh b/modules/tensorflow/src/main/sh/ignite-tf.sh
new file mode 100755
index 0000000..fd3e02c
--- /dev/null
+++ b/modules/tensorflow/src/main/sh/ignite-tf.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SCRIPT_PATH="$( cd "$(dirname "$0")" ; pwd -P )"
+java -Xmx4G -DIGNITE_QUIET=false -cp "$SCRIPT_PATH:$SCRIPT_PATH/lib/*" org.apache.ignite.tensorflow.submitter.JobSubmitter "$@"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/main/sh/logback.xml
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/sh/logback.xml b/modules/tensorflow/src/main/sh/logback.xml
new file mode 100644
index 0000000..816b5e6
--- /dev/null
+++ b/modules/tensorflow/src/main/sh/logback.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ Logback configuration file.
+-->
+<configuration>
+
+ <appender name="FILE" class="ch.qos.logback.core.FileAppender">
+ <file>ignite-tf.log</file>
+ <append>false</append>
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="warn">
+ <appender-ref ref="FILE" />
+ </root>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e884e5a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManagerTest.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManagerTest.java b/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManagerTest.java
index 7d917e7..faa2b6b 100644
--- a/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManagerTest.java
+++ b/modules/tensorflow/src/test/java/org/apache/ignite/tensorflow/core/longrunning/LongRunningProcessManagerTest.java
@@ -17,13 +17,11 @@
package org.apache.ignite.tensorflow.core.longrunning;
-import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
-import java.util.function.Supplier;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCluster;
import org.apache.ignite.IgniteCompute;
@@ -67,7 +65,7 @@ public class LongRunningProcessManagerTest {
List<LongRunningProcess> list = Collections.singletonList(new LongRunningProcess(nodeId, () -> {}));
- LongRunningProcessManager mgr = new LongRunningProcessManager((Supplier<Ignite> & Serializable)() -> ignite);
+ LongRunningProcessManager mgr = new LongRunningProcessManager(ignite);
Map<UUID, List<UUID>> res = mgr.start(list);
assertEquals(1, res.size());
@@ -97,7 +95,7 @@ public class LongRunningProcessManagerTest {
Map<UUID, List<UUID>> procIds = new HashMap<>();
procIds.put(nodeId, Collections.singletonList(procId));
- LongRunningProcessManager mgr = new LongRunningProcessManager((Supplier<Ignite> & Serializable)() -> ignite);
+ LongRunningProcessManager mgr = new LongRunningProcessManager(ignite);
Map<UUID, List<LongRunningProcessStatus>> res = mgr.ping(procIds);
assertEquals(1, res.size());
@@ -127,7 +125,7 @@ public class LongRunningProcessManagerTest {
Map<UUID, List<UUID>> procIds = new HashMap<>();
procIds.put(nodeId, Collections.singletonList(procId));
- LongRunningProcessManager mgr = new LongRunningProcessManager((Supplier<Ignite> & Serializable)() -> ignite);
+ LongRunningProcessManager mgr = new LongRunningProcessManager(ignite);
Map<UUID, List<LongRunningProcessStatus>> res = mgr.stop(procIds, true);
assertEquals(1, res.size());
@@ -157,7 +155,7 @@ public class LongRunningProcessManagerTest {
Map<UUID, List<UUID>> procIds = new HashMap<>();
procIds.put(nodeId, Collections.singletonList(procId));
- LongRunningProcessManager mgr = new LongRunningProcessManager((Supplier<Ignite> & Serializable)() -> ignite);
+ LongRunningProcessManager mgr = new LongRunningProcessManager(ignite);
Map<UUID, List<LongRunningProcessStatus>> res = mgr.clear(procIds);
assertEquals(1, res.size());
[22/50] [abbrv] ignite git commit: IGNITE-9130 Test fixed
ZookeeperDiscoverySpiTest.testDuplicatedNodeId. - Fixes #4458.
Posted by av...@apache.org.
IGNITE-9130 Test fixed ZookeeperDiscoverySpiTest.testDuplicatedNodeId. - Fixes #4458.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8bbcda35
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8bbcda35
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8bbcda35
Branch: refs/heads/ignite-8446
Commit: 8bbcda3598487a6784da86655c5ec607c570f0e2
Parents: 1f06420
Author: Vitaliy Biryukov <Bi...@gmail.com>
Authored: Tue Jul 31 17:37:04 2018 +0300
Committer: Dmitriy Pavlov <dp...@apache.org>
Committed: Tue Jul 31 17:37:04 2018 +0300
----------------------------------------------------------------------
.../spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8bbcda35/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index cc80a72..f38a097 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -2554,10 +2554,10 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
}
}, IgniteCheckedException.class, null);
- IgniteSpiException spiErr = X.cause(err, IgniteSpiException.class);
+ assertTrue(err instanceof IgniteCheckedException);
- assertNotNull(spiErr);
- assertTrue(spiErr.getMessage().contains("Node with the same ID already exists"));
+ assertTrue(err.getMessage().contains("Failed to start processor:")
+ || err.getMessage().contains("Failed to start manager:"));
nodeId = null;
[14/50] [abbrv] ignite git commit: IGNITE-6846 Add metrics for entry
processor invocations. - Fixes #3148.
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
index d930c6b..8f3f2ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
@@ -1506,6 +1506,17 @@ public class PlatformCache extends PlatformAbstractTarget {
writer.writeLong(metrics.getCacheSize());
writer.writeLong(metrics.getRebalancedKeys());
writer.writeLong(metrics.getEstimatedRebalancingKeys());
+ writer.writeLong(metrics.getEntryProcessorPuts());
+ writer.writeFloat(metrics.getEntryProcessorAverageInvocationTime());
+ writer.writeLong(metrics.getEntryProcessorInvocations());
+ writer.writeFloat(metrics.getEntryProcessorMaxInvocationTime());
+ writer.writeFloat(metrics.getEntryProcessorMinInvocationTime());
+ writer.writeLong(metrics.getEntryProcessorReadOnlyInvocations());
+ writer.writeFloat(metrics.getEntryProcessorHitPercentage());
+ writer.writeLong(metrics.getEntryProcessorHits());
+ writer.writeLong(metrics.getEntryProcessorMisses());
+ writer.writeFloat(metrics.getEntryProcessorMissPercentage());
+ writer.writeLong(metrics.getEntryProcessorRemovals());
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java
index 721989b..eb4d2d5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache;
+import com.google.common.collect.ImmutableMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
@@ -27,9 +28,14 @@ import java.util.concurrent.TimeUnit;
import javax.cache.expiry.Duration;
import javax.cache.expiry.ExpiryPolicy;
import javax.cache.expiry.TouchedExpiryPolicy;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
+import javax.cache.processor.MutableEntry;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheEntryProcessor;
import org.apache.ignite.cache.CacheMetrics;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.configuration.CacheConfiguration;
@@ -49,6 +55,51 @@ public abstract class GridCacheAbstractMetricsSelfTest extends GridCacheAbstract
/** */
private static final int KEY_CNT = 500;
+ /** Entry processor, performing removal. */
+ private final CacheEntryProcessor<Integer, Integer, Object> removingProcessor =
+ new CacheEntryProcessor<Integer, Integer, Object>() {
+ @Override
+ public Object process(
+ MutableEntry<Integer, Integer> entry,
+ Object... arguments
+ ) throws EntryProcessorException {
+
+ entry.remove();
+
+ return null;
+ }
+ };
+
+ /** Entry processor, performing reading. */
+ private final CacheEntryProcessor<Integer, Integer, Object> readingProcessor =
+ new CacheEntryProcessor<Integer, Integer, Object>() {
+ @Override
+ public Object process(
+ MutableEntry<Integer, Integer> entry,
+ Object... arguments
+ ) throws EntryProcessorException {
+
+ entry.getValue();
+
+ return null;
+ }
+ };
+
+ /** Entry processor, performing updating. */
+ private final CacheEntryProcessor<Integer, Integer, Object> updatingProcessor =
+ new CacheEntryProcessor<Integer, Integer, Object>() {
+ @Override
+ public Object process(
+ MutableEntry<Integer, Integer> entry,
+ Object... arguments
+ ) throws EntryProcessorException {
+
+ entry.setValue(1);
+
+ return null;
+ }
+ };
+
/** {@inheritDoc} */
@Override protected boolean swapEnabled() {
return false;
@@ -134,6 +185,47 @@ public abstract class GridCacheAbstractMetricsSelfTest extends GridCacheAbstract
for (int i = 0; i < KEY_CNT; i++)
jcache.put(i, i);
+ // Invoke update on cache.
+ for (int i = 0; i < KEY_CNT; i++)
+ jcache.invoke(i, new CacheEntryProcessor<Object, Object, Object>() {
+ @Override
+ public Object process(MutableEntry<Object, Object> entry,
+ Object... arguments) throws EntryProcessorException {
+
+ Object key = entry.getKey();
+
+ entry.setValue(key);
+
+ return null;
+ }
+ });
+
+ // Read-only invoke on cache.
+ for (int i = 0; i < KEY_CNT; i++)
+ jcache.invoke(i, new CacheEntryProcessor<Object, Object, Object>() {
+ @Override
+ public Object process(MutableEntry<Object, Object> entry,
+ Object... arguments) throws EntryProcessorException {
+
+ entry.getKey();
+
+ return null;
+ }
+ });
+
+ // Remove invoke on cache.
+ for (int i = 0; i < KEY_CNT; i++)
+ jcache.invoke(i, new CacheEntryProcessor<Object, Object, Object>() {
+ @Override
+ public Object process(MutableEntry<Object, Object> entry,
+ Object... arguments) throws EntryProcessorException {
+
+ entry.remove();
+
+ return null;
+ }
+ });
+
// Get from cache.
for (int i = 0; i < KEY_CNT; i++)
jcache.get(i);
@@ -156,6 +248,14 @@ public abstract class GridCacheAbstractMetricsSelfTest extends GridCacheAbstract
assertEquals(m.getAveragePutTime(), 0f);
assertEquals(m.getAverageTxCommitTime(), 0f);
assertEquals(m.getAverageTxRollbackTime(), 0f);
+
+ assertEquals(m.getEntryProcessorPuts(), 0);
+ assertEquals(m.getEntryProcessorRemovals(), 0);
+ assertEquals(m.getEntryProcessorReadOnlyInvocations(), 0);
+ assertEquals(m.getEntryProcessorMinInvocationTime(), 0f);
+ assertEquals(m.getEntryProcessorMaxInvocationTime(), 0f);
+ assertEquals(m.getEntryProcessorAverageInvocationTime(), 0f);
+ assertEquals(m.getEntryProcessorInvocations(), 0);
}
}
@@ -934,4 +1034,321 @@ public abstract class GridCacheAbstractMetricsSelfTest extends GridCacheAbstract
assertEquals(0, entry.expireTime());
}
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testInvocationRemovesOnEmptyCache() throws IgniteCheckedException {
+ testInvocationRemoves(true);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testInvocationRemoves() throws IgniteCheckedException {
+ testInvocationRemoves(false);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ private void testInvocationRemoves(boolean emptyCache) throws IgniteCheckedException {
+ IgniteCache<Integer, Integer> cache0 = grid(0).cache(DEFAULT_CACHE_NAME);
+
+ final Integer key = primaryKey(cache0);
+
+ if (emptyCache)
+ cache0.remove(key);
+ else
+ cache0.put(key, 0);
+
+ cache0.invoke(key, removingProcessor);
+
+ assertEquals(1, cache0.localMetrics().getEntryProcessorRemovals());
+
+ if (emptyCache) {
+ assertEquals(1, cache0.localMetrics().getEntryProcessorMisses());
+
+ assertEquals(100f, cache0.localMetrics().getEntryProcessorMissPercentage());
+ assertEquals(0f, cache0.localMetrics().getEntryProcessorHitPercentage());
+ }
+ else {
+ assertEquals(1, cache0.localMetrics().getEntryProcessorHits());
+
+ assertEquals(0f, cache0.localMetrics().getEntryProcessorMissPercentage());
+ assertEquals(100f, cache0.localMetrics().getEntryProcessorHitPercentage());
+ }
+
+ for (int i = 1; i < gridCount(); i++) {
+ Ignite ignite = ignite(i);
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+ if (affinity(cache).isPrimaryOrBackup(ignite.cluster().localNode(), key))
+ assertEquals(1, cache.localMetrics().getEntryProcessorRemovals());
+ }
+
+ assertEquals(1, cache0.localMetrics().getEntryProcessorInvocations());
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testUpdateInvocationsOnEmptyCache() throws IgniteCheckedException {
+ testUpdateInvocations(true);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testUpdateInvocations() throws IgniteCheckedException {
+ testUpdateInvocations(false);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ private void testUpdateInvocations(final boolean emptyCache) throws IgniteCheckedException {
+ IgniteCache<Integer, Integer> cache0 = grid(0).cache(DEFAULT_CACHE_NAME);
+
+ final Integer key = primaryKey(jcache(0));
+
+ if (emptyCache)
+ cache0.remove(key);
+ else
+ cache0.put(key, 0);
+
+ cache0.invoke(key, updatingProcessor);
+
+ assertEquals(1, cache0.localMetrics().getEntryProcessorPuts());
+
+ if (emptyCache) {
+ assertEquals(1, cache0.localMetrics().getEntryProcessorMisses());
+
+ assertEquals(100f, cache0.localMetrics().getEntryProcessorMissPercentage());
+ assertEquals(0f, cache0.localMetrics().getEntryProcessorHitPercentage());
+ }
+ else {
+ assertEquals(1, cache0.localMetrics().getEntryProcessorHits());
+
+ assertEquals(0f, cache0.localMetrics().getEntryProcessorMissPercentage());
+ assertEquals(100f, cache0.localMetrics().getEntryProcessorHitPercentage());
+ }
+
+ for (int i = 1; i < gridCount(); i++) {
+ Ignite ignite = ignite(i);
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+ if (affinity(cache).isPrimaryOrBackup(ignite.cluster().localNode(), key))
+ assertEquals(1, cache.localMetrics().getEntryProcessorPuts());
+ }
+
+ assertEquals(1, cache0.localMetrics().getEntryProcessorInvocations());
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testReadOnlyInvocationsOnEmptyCache() throws IgniteCheckedException {
+ testReadOnlyInvocations(true);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testReadOnlyInvocations() throws IgniteCheckedException {
+ testReadOnlyInvocations(false);
+ }
+
+ /**
+ * @param emptyCache Empty cache.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void testReadOnlyInvocations(final boolean emptyCache) throws IgniteCheckedException {
+ IgniteCache<Integer, Integer> cache0 = grid(0).cache(DEFAULT_CACHE_NAME);
+
+ final Integer key = primaryKey(jcache(0));
+
+ if (emptyCache)
+ cache0.remove(key);
+ else
+ cache0.put(key, 0);
+
+ cache0.invoke(key, readingProcessor);
+
+ assertEquals(1, cache0.localMetrics().getEntryProcessorReadOnlyInvocations());
+
+ if (emptyCache) {
+ assertEquals(1, cache0.localMetrics().getEntryProcessorMisses());
+
+ assertEquals(100f, cache0.localMetrics().getEntryProcessorMissPercentage());
+ assertEquals(0f, cache0.localMetrics().getEntryProcessorHitPercentage());
+ }
+ else {
+ assertEquals(1, cache0.localMetrics().getEntryProcessorHits());
+
+ assertEquals(0f, cache0.localMetrics().getEntryProcessorMissPercentage());
+ assertEquals(100f, cache0.localMetrics().getEntryProcessorHitPercentage());
+ }
+
+ for (int i = 1; i < gridCount(); i++)
+ assertEquals(0, jcache(i).localMetrics().getEntryProcessorReadOnlyInvocations());
+
+ assertEquals(1, cache0.localMetrics().getEntryProcessorInvocations());
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testInvokeAvgTime() throws IgniteCheckedException {
+ IgniteCache<Integer, Integer> cache0 = grid(0).cache(DEFAULT_CACHE_NAME);
+
+ float averageTime = cache0.localMetrics().getEntryProcessorAverageInvocationTime();
+
+ assertEquals(0.0, averageTime, 0.001f);
+
+ final Integer key = primaryKey(cache0);
+
+ cache0.invoke(key, new CacheEntryProcessor<Integer, Integer, Object>() {
+ @Override
+ public Object process(MutableEntry<Integer, Integer> entry,
+ Object... arguments) throws EntryProcessorException {
+
+ entry.setValue(1);
+
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new EntryProcessorException(e);
+ }
+
+ return null;
+ }
+ });
+
+ averageTime = cache0.localMetrics().getEntryProcessorAverageInvocationTime();
+
+ assertTrue(averageTime > 0.0);
+
+ float maxTime = cache0.localMetrics().getEntryProcessorMaxInvocationTime();
+ float minTime = cache0.localMetrics().getEntryProcessorMinInvocationTime();
+
+ assertTrue(maxTime > 0.0);
+ assertEquals(maxTime, minTime, 0.001f);
+
+ cache0.invoke(key, new CacheEntryProcessor<Integer, Integer, Object>() {
+ @Override
+ public Object process(MutableEntry<Integer, Integer> entry,
+ Object... arguments) throws EntryProcessorException {
+
+ entry.setValue(1);
+
+ try {
+ Thread.sleep(200);
+ } catch (InterruptedException e) {
+ throw new EntryProcessorException(e);
+ }
+
+ return null;
+ }
+ });
+
+ maxTime = cache0.localMetrics().getEntryProcessorMaxInvocationTime();
+ minTime = cache0.localMetrics().getEntryProcessorMinInvocationTime();
+ averageTime = cache0.localMetrics().getEntryProcessorAverageInvocationTime();
+
+ assertTrue(maxTime > averageTime && averageTime > minTime);
+ assertEquals(2, cache0.localMetrics().getEntryProcessorInvocations());
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testInvokeAsyncAvgTime() throws IgniteCheckedException {
+ IgniteCache<Integer, Integer> cache = grid(0).cache(DEFAULT_CACHE_NAME);
+
+ assertEquals(0.0, cache.localMetrics().getEntryProcessorAverageInvocationTime(), 0.001f);
+
+ final Integer key = primaryKey(cache);
+
+ cache.invokeAsync(key, updatingProcessor).get();
+
+ U.sleep(100);
+
+ assertTrue(cache.localMetrics().getEntryProcessorAverageInvocationTime() > 0.0);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testInvokeAllAvgTime() throws IgniteCheckedException {
+ IgniteCache<Integer, Integer> cache = grid(0).cache(DEFAULT_CACHE_NAME);
+
+ assertEquals(0.0, cache.localMetrics().getEntryProcessorAverageInvocationTime(), 0.001f);
+
+ cache.invokeAll(ImmutableMap.of(0, updatingProcessor,
+ 1, readingProcessor,
+ 2, removingProcessor));
+
+ U.sleep(100);
+
+ assertTrue(cache.localMetrics().getEntryProcessorAverageInvocationTime() > 0.0);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testInvokeAllAsyncAvgTime() throws IgniteCheckedException {
+ IgniteCache<Integer, Integer> cache = grid(0).cache(DEFAULT_CACHE_NAME);
+
+ assertEquals(0.0, cache.localMetrics().getEntryProcessorAverageInvocationTime(), 0.001f);
+
+ Map<Integer, EntryProcessorResult<Object>> invokeFut = cache.invokeAllAsync(
+ ImmutableMap.of(0, updatingProcessor,
+ 1, readingProcessor,
+ 2, removingProcessor)).get();
+
+ U.sleep(100);
+
+ assertTrue(cache.localMetrics().getEntryProcessorAverageInvocationTime() > 0.0);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testInvokeAllMultipleKeysAvgTime() throws IgniteCheckedException {
+ IgniteCache<Integer, Integer> cache = grid(0).cache(DEFAULT_CACHE_NAME);
+
+ Set<Integer> keys = new HashSet<>();
+ keys.add(1);
+ keys.add(2);
+
+ assertEquals(0.0, cache.localMetrics().getEntryProcessorAverageInvocationTime(), 0.001f);
+
+ cache.invokeAll(keys, updatingProcessor);
+
+ U.sleep(100);
+
+ assertTrue(cache.localMetrics().getEntryProcessorAverageInvocationTime() > 0.0);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testInvokeAllAsyncMultipleKeysAvgTime() throws IgniteCheckedException {
+ IgniteCache<Integer, Integer> cache = grid(0).cache(DEFAULT_CACHE_NAME);
+
+ Set<Integer> keys = new HashSet<>();
+ keys.add(1);
+ keys.add(2);
+
+ assertEquals(0.0, cache.localMetrics().getEntryProcessorAverageInvocationTime(), 0.001f);
+
+ cache.invokeAllAsync(keys, updatingProcessor).get();
+
+ U.sleep(100);
+
+ assertTrue(cache.localMetrics().getEntryProcessorAverageInvocationTime() > 0.0);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index daf96c4..99abd70 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -478,7 +478,8 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
@Nullable CacheEntryPredicate[] filter,
boolean intercept,
UUID subjId,
- String taskName)
+ String taskName,
+ boolean transformOp)
throws IgniteCheckedException, GridCacheEntryRemovedException {
return new GridTuple3<>(false, null, null);
}
@@ -512,8 +513,9 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
String taskName,
@Nullable CacheObject prevVal,
@Nullable Long updateCntr,
- @Nullable GridDhtAtomicAbstractUpdateFuture fut) throws IgniteCheckedException,
- GridCacheEntryRemovedException {
+ @Nullable GridDhtAtomicAbstractUpdateFuture fut,
+ boolean transformOp)
+ throws IgniteCheckedException, GridCacheEntryRemovedException {
assert false;
return null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearAtomicMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearAtomicMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearAtomicMetricsSelfTest.java
new file mode 100644
index 0000000..70cebff
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearAtomicMetricsSelfTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+
+/**
+ * Atomic cache metrics test.
+ */
+public class GridCacheNearAtomicMetricsSelfTest extends GridCacheNearMetricsSelfTest {
+ /** {@inheritDoc} */
+ @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception {
+ CacheConfiguration ccfg = super.cacheConfiguration(igniteInstanceName);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+ return ccfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
index 341b1fe..70b06f3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearMetricsSelfTest.java
@@ -17,8 +17,11 @@
package org.apache.ignite.internal.processors.cache.distributed.near;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheEntryProcessor;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.cluster.ClusterNode;
@@ -446,4 +449,389 @@ public class GridCacheNearMetricsSelfTest extends GridCacheAbstractSelfTest {
}
}
}
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCreateReadRemoveInvokesFromPrimary() throws Exception {
+ Ignite g0 = grid(0);
+
+ IgniteCache<Integer, Integer> cache0 = g0.cache(DEFAULT_CACHE_NAME);
+
+ int key = primaryKey(cache0);
+
+ setValue1ByEntryProcessor(cache0, key);
+
+ readKeyByEntryProcessor(cache0, key);
+
+ removeKeyByEntryProcessor(cache0, key);
+
+ for (int j = 0; j < gridCount(); j++) {
+ Ignite g = grid(j);
+
+ IgniteCache<Object, Object> jcache = g.cache(DEFAULT_CACHE_NAME);
+
+ if (affinity(jcache).isPrimaryOrBackup(g.cluster().localNode(), key))
+ assertEquals(1, jcache.localMetrics().getEntryProcessorPuts());
+ else
+ assertEquals(0, jcache.localMetrics().getEntryProcessorPuts());
+
+ if (affinity(jcache).isPrimary(g.cluster().localNode(), key)) {
+ assertEquals(1, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(3, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(1, jcache.localMetrics().getEntryProcessorMisses());
+ assertEquals(2, jcache.localMetrics().getEntryProcessorHits());
+
+ assertEquals((float) 1 / 3 * 100.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ assertEquals((float) 2 / 3 * 100.0f, jcache.localMetrics().getEntryProcessorHitPercentage(), 0.001f);
+ }
+ else if (affinity(jcache).isBackup(g.cluster().localNode(), key)) {
+ assertEquals(1, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(0, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(2, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(1, jcache.localMetrics().getEntryProcessorMisses());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorHits());
+
+ assertEquals(50.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ assertEquals(50.0f, jcache.localMetrics().getEntryProcessorHitPercentage(), 0.001f);
+ }
+ else
+ assertNoMetricsChanged(jcache);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCreateReadRemoveInvokesFromBackup() throws Exception {
+ Ignite g0 = grid(0);
+
+ IgniteCache<Integer, Integer> cache0 = g0.cache(DEFAULT_CACHE_NAME);
+
+ int key = backupKey(cache0);
+
+ setValue1ByEntryProcessor(cache0, key);
+
+ readKeyByEntryProcessor(cache0, key);
+
+ removeKeyByEntryProcessor(cache0, key);
+
+ for (int j = 0; j < gridCount(); j++) {
+ Ignite g = grid(j);
+
+ IgniteCache<Object, Object> jcache = g.cache(DEFAULT_CACHE_NAME);
+
+ if (affinity(jcache).isPrimaryOrBackup(g.cluster().localNode(), key))
+ assertEquals(1, jcache.localMetrics().getEntryProcessorPuts());
+ else
+ assertEquals(0, jcache.localMetrics().getEntryProcessorPuts());
+
+ if (affinity(jcache).isPrimary(g.cluster().localNode(), key)) {
+ assertEquals(1, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(3, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(1, jcache.localMetrics().getEntryProcessorMisses());
+ assertEquals(2, jcache.localMetrics().getEntryProcessorHits());
+
+ assertEquals((float) 1 / 3 * 100.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ assertEquals((float) 2 / 3 * 100.0f, jcache.localMetrics().getEntryProcessorHitPercentage(), 0.001f);
+ }
+ else if (affinity(jcache).isBackup(g.cluster().localNode(), key)) {
+ assertEquals(1, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(0, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(2, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(1, jcache.localMetrics().getEntryProcessorMisses());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorHits());
+
+ assertEquals(50.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ assertEquals(50.0f, jcache.localMetrics().getEntryProcessorHitPercentage(), 0.001f);
+ }
+ else
+ assertNoMetricsChanged(jcache);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCreateReadRemoveInvokesFromNear() throws Exception {
+ Ignite g0 = grid(0);
+
+ IgniteCache<Integer, Integer> cache0 = g0.cache(DEFAULT_CACHE_NAME);
+
+ int key;
+
+ for (int i = 0; ; i++) {
+ if (!affinity(cache0).isPrimaryOrBackup(g0.cluster().localNode(), i)) {
+ setValue1ByEntryProcessor(cache0, i);
+
+ readKeyByEntryProcessor(cache0, i);
+
+ removeKeyByEntryProcessor(cache0, i);
+
+ key = i;
+
+ break;
+ }
+ }
+
+ for (int j = 0; j < gridCount(); j++) {
+ Ignite g = grid(j);
+
+ IgniteCache<Object, Object> jcache = g.cache(DEFAULT_CACHE_NAME);
+
+ assertEquals(1, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorMisses());
+
+ if (affinity(jcache).isPrimary(g.cluster().localNode(), key)) {
+ assertEquals(1, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(3, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(2, jcache.localMetrics().getEntryProcessorHits());
+
+ assertEquals((float) 1 / 3 * 100.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ assertEquals((float) 2 / 3 * 100.0f, jcache.localMetrics().getEntryProcessorHitPercentage(), 0.001f);
+ } else {
+ assertEquals(0, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(2, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(1, jcache.localMetrics().getEntryProcessorHits());
+
+ assertEquals(50.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ assertEquals(50.0f, jcache.localMetrics().getEntryProcessorHitPercentage(), 0.001f);
+ }
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReadRemoveInvokesFromPrimary() throws Exception {
+ Ignite g0 = grid(0);
+
+ IgniteCache<Integer, Integer> cache0 = g0.cache(DEFAULT_CACHE_NAME);
+
+ int key = primaryKey(cache0);
+
+ readKeyByEntryProcessor(cache0, key);
+
+ removeKeyByEntryProcessor(cache0, key);
+
+ for (int j = 0; j < gridCount(); j++) {
+ Ignite g = grid(j);
+
+ IgniteCache<Object, Object> jcache = g.cache(DEFAULT_CACHE_NAME);
+
+ if (affinity(jcache).isPrimary(g.cluster().localNode(), key)) {
+ assertEquals(0, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+
+ assertEquals(2, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(2, jcache.localMetrics().getEntryProcessorMisses());
+
+ assertEquals(100.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ }
+ else if (affinity(jcache).isBackup(g.cluster().localNode(), key)) {
+ assertEquals(0, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(0, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(1, jcache.localMetrics().getEntryProcessorMisses());
+
+ assertEquals(100.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ }
+ else
+ assertNoMetricsChanged(jcache);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReadRemoveInvokesFromBackup() throws Exception {
+ Ignite g0 = grid(0);
+
+ IgniteCache<Integer, Integer> cache0 = g0.cache(DEFAULT_CACHE_NAME);
+
+ int key = backupKey(cache0);
+
+ readKeyByEntryProcessor(cache0, key);
+
+ removeKeyByEntryProcessor(cache0, key);
+
+ for (int j = 0; j < gridCount(); j++) {
+ Ignite g = grid(j);
+
+ IgniteCache<Object, Object> jcache = g.cache(DEFAULT_CACHE_NAME);
+
+ if (affinity(jcache).isPrimary(g.cluster().localNode(), key)) {
+ assertEquals(0, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(2, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(2, jcache.localMetrics().getEntryProcessorMisses());
+
+ assertEquals(100.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ }
+ else if (affinity(jcache).isBackup(g.cluster().localNode(), key)) {
+
+ assertEquals(0, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(0, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(1, jcache.localMetrics().getEntryProcessorMisses());
+
+ assertEquals(100.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ }
+ else
+ assertNoMetricsChanged(jcache);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReadRemoveInvokesFromNear() throws Exception {
+ Ignite g0 = grid(0);
+
+ IgniteCache<Integer, Integer> cache0 = g0.cache(DEFAULT_CACHE_NAME);
+
+ int key;
+
+ for (int i = 0; ; i++) {
+ if (!affinity(cache0).isPrimaryOrBackup(g0.cluster().localNode(), i)) {
+ readKeyByEntryProcessor(cache0, i);
+
+ removeKeyByEntryProcessor(cache0, i);
+
+ key = i;
+
+ break;
+ }
+ }
+
+ for (int j = 0; j < gridCount(); j++) {
+ Ignite g = grid(j);
+
+ IgniteCache<Object, Object> jcache = g.cache(DEFAULT_CACHE_NAME);
+
+ if (affinity(jcache).isPrimary(g.cluster().localNode(), key)) {
+ assertEquals(0, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(2, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(2, jcache.localMetrics().getEntryProcessorMisses());
+
+ assertEquals(100.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ }
+ else if (affinity(jcache).isBackup(g.cluster().localNode(), key)) {
+ assertEquals(0, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(0, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(1, jcache.localMetrics().getEntryProcessorMisses());
+
+ assertEquals(100.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ }
+ else {
+ assertEquals(0, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(0, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(1, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(1, jcache.localMetrics().getEntryProcessorMisses());
+
+ assertEquals(100.0f, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ }
+ }
+ }
+
+ /**
+ * Checks no metrics changed in cache.
+ *
+ * @param jcache Cache to be checked.
+ */
+ private void assertNoMetricsChanged(IgniteCache<Object, Object> jcache) {
+ assertEquals(0, jcache.localMetrics().getEntryProcessorPuts());
+ assertEquals(0, jcache.localMetrics().getEntryProcessorRemovals());
+ assertEquals(0, jcache.localMetrics().getEntryProcessorReadOnlyInvocations());
+ assertEquals(0, jcache.localMetrics().getEntryProcessorInvocations());
+
+ assertEquals(0, jcache.localMetrics().getEntryProcessorMisses());
+ assertEquals(0, jcache.localMetrics().getEntryProcessorHits());
+
+ assertEquals(0, jcache.localMetrics().getEntryProcessorMissPercentage(), 0.001f);
+ assertEquals(0, jcache.localMetrics().getEntryProcessorHitPercentage(), 0.001f);
+ }
+
+ /**
+ * Invokes entry processor, which removes key from cache.
+ *
+ * @param cache Cache.
+ * @param key Key.
+ */
+ private void removeKeyByEntryProcessor(IgniteCache<Integer, Integer> cache, int key) {
+ cache.invoke(key, new CacheEntryProcessor<Integer, Integer, Object>() {
+ @Override public Object process(MutableEntry<Integer, Integer> entry,
+ Object... arguments) throws EntryProcessorException {
+ entry.remove();
+
+ return null;
+ }
+ });
+ }
+
+ /**
+ * Invokes entry processor, which reads key from cache.
+ *
+ * @param cache Cache.
+ * @param key Key.
+ */
+ private void readKeyByEntryProcessor(IgniteCache<Integer, Integer> cache, int key) {
+ cache.invoke(key, new CacheEntryProcessor<Integer, Integer, Object>() {
+ @Override public Object process(MutableEntry<Integer, Integer> entry,
+ Object... arguments) throws EntryProcessorException {
+ entry.getValue();
+
+ return null;
+ }
+ });
+ }
+
+ /**
+ * Invokes entry processor, which sets value "1" for key into cache.
+ *
+ * @param cache Cache.
+ * @param key Key.
+ */
+ private void setValue1ByEntryProcessor(IgniteCache<Integer, Integer> cache, int key) {
+ cache.invoke(key, new CacheEntryProcessor<Integer, Integer, Object>() {
+ @Override public Object process(MutableEntry<Integer, Integer> entry,
+ Object... arguments) throws EntryProcessorException {
+ entry.setValue(1);
+
+ return null;
+ }
+ });
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java b/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java
index fe61e35..f4c1642 100644
--- a/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java
+++ b/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java
@@ -483,5 +483,60 @@ public class PlatformCacheWriteMetricsTask extends ComputeTaskAdapter<Long, Obje
@Override public long getEstimatedRebalancingKeys() {
return 67;
}
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorPuts() {
+ return 68;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorReadOnlyInvocations() {
+ return 69;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorInvocations() {
+ return 70;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorHits() {
+ return 71;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorHitPercentage() {
+ return 72;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMissPercentage() {
+ return 73;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorMisses() {
+ return 74;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorAverageInvocationTime() {
+ return 75;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMinInvocationTime() {
+ return 76;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEntryProcessorMaxInvocationTime() {
+ return 77;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getEntryProcessorRemovals() {
+ return 78;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
index ac4b512..6f62ffb 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
@@ -29,6 +29,7 @@ import org.apache.ignite.internal.processors.cache.GridEvictionPolicyMBeansTest;
import org.apache.ignite.internal.processors.cache.OffheapCacheMetricsForClusterGroupSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPartitionedMetricsSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPartitionedTckMetricsSelfTestImpl;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearAtomicMetricsSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearMetricsSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedHitsAndMissesSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMetricsSelfTest;
@@ -54,6 +55,7 @@ public class IgniteCacheMetricsSelfTestSuite extends TestSuite {
suite.addTestSuite(GridCacheLocalMetricsSelfTest.class);
suite.addTestSuite(GridCacheLocalAtomicMetricsNoReadThroughSelfTest.class);
suite.addTestSuite(GridCacheNearMetricsSelfTest.class);
+ suite.addTestSuite(GridCacheNearAtomicMetricsSelfTest.class);
suite.addTestSuite(GridCacheReplicatedMetricsSelfTest.class);
suite.addTestSuite(GridCachePartitionedMetricsSelfTest.class);
suite.addTestSuite(GridCachePartitionedHitsAndMissesSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheMetricsTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheMetricsTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheMetricsTest.cs
index 129b4b5..d5f8d9e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheMetricsTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheMetricsTest.cs
@@ -219,6 +219,17 @@ namespace Apache.Ignite.Core.Tests.Cache
Assert.AreEqual(true, metrics.IsWriteThrough);
Assert.AreEqual(true, metrics.IsValidForReading);
Assert.AreEqual(true, metrics.IsValidForWriting);
+ Assert.AreEqual(68, metrics.EntryProcessorPuts);
+ Assert.AreEqual(69, metrics.EntryProcessorReadOnlyInvocations);
+ Assert.AreEqual(70, metrics.EntryProcessorInvocations);
+ Assert.AreEqual(71, metrics.EntryProcessorHits);
+ Assert.AreEqual(72, metrics.EntryProcessorHitPercentage);
+ Assert.AreEqual(73, metrics.EntryProcessorMissPercentage);
+ Assert.AreEqual(74, metrics.EntryProcessorMisses);
+ Assert.AreEqual(75, metrics.EntryProcessorAverageInvocationTime);
+ Assert.AreEqual(76, metrics.EntryProcessorMinInvocationTime);
+ Assert.AreEqual(77, metrics.EntryProcessorMaxInvocationTime);
+ Assert.AreEqual(78, metrics.EntryProcessorRemovals);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
index e0e7301..53be9ef 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
@@ -672,5 +672,93 @@ namespace Apache.Ignite.Core.Cache
/// Number of estimated keys to rebalance.
/// </returns>
long EstimatedRebalancingKeys { get; }
+
+ /// <summary>
+ /// The total number of cache invocations, caused update.
+ /// </summary>
+ /// <returns>
+ /// The number of invocation updates.
+ /// </returns>
+ long EntryProcessorPuts { get; }
+
+ /// <summary>
+ /// The total number of cache invocations, caused removal.
+ /// </summary>
+ /// <returns>
+ /// The number of invocation removals.
+ /// </returns>
+ long EntryProcessorRemovals { get; }
+
+ /// <summary>
+ /// The total number of cache invocations, caused no updates.
+ /// </summary>
+ /// <returns>
+ /// The number of read-only invocations.
+ /// </returns>
+ long EntryProcessorReadOnlyInvocations { get; }
+
+ /// <summary>
+ /// The total number of cache invocations.
+ /// </summary>
+ /// <returns>
+ /// The number of cache invocations.
+ /// </returns>
+ long EntryProcessorInvocations { get; }
+
+ /// <summary>
+ /// The total number of invocations on keys, which exist in cache.
+ /// </summary>
+ /// <returns>
+ /// The number of cache invocation hits.
+ /// </returns>
+ long EntryProcessorHits { get; }
+
+ /// <summary>
+ /// The percentage of invocations on keys, which exist in cache.
+ /// </summary>
+ /// <returns>
+ /// The percentage of successful invocation hits.
+ /// </returns>
+ float EntryProcessorHitPercentage { get; }
+
+ /// <summary>
+ /// The total number of invocations on keys, which don't exist in cache.
+ /// </summary>
+ /// <returns>
+ /// The number of cache invocation misses.
+ /// </returns>
+ long EntryProcessorMisses { get; }
+
+ /// <summary>
+ /// The percentage of invocations on keys, which don't exist in cache.
+ /// </summary>
+ /// <returns>
+ /// The percentage of invocation misses.
+ /// </returns>
+ float EntryProcessorMissPercentage { get; }
+
+ /// <summary>
+ /// The mean time to execute cache invokes.
+ /// </summary>
+ /// <returns>
+ /// The time in µs.
+ /// </returns>
+ float EntryProcessorAverageInvocationTime { get; }
+
+ /// <summary>
+ /// So far, the minimum time to execute cache invokes.
+ /// </summary>
+ /// <returns>
+ /// The time in µs.
+ /// </returns>
+ float EntryProcessorMinInvocationTime { get; }
+
+ /// <summary>
+ /// So far, the maximum time to execute cache invokes.
+ /// </summary>
+ /// <returns>
+ /// The time in µs.
+ /// </returns>
+ float EntryProcessorMaxInvocationTime { get; }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e8669af/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
index be6980d..f34fa3e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
@@ -253,6 +253,39 @@ namespace Apache.Ignite.Core.Impl.Cache
/** */
private readonly long _estimatedRebalancedKeys;
+ /** */
+ private readonly long _entryProcessorPuts;
+
+ /** */
+ private readonly float _entryProcessorAverageInvocationTime;
+
+ /** */
+ private readonly long _entryProcessorInvocations;
+
+ /** */
+ private readonly float _entryProcessorMaxInvocationTime;
+
+ /** */
+ private readonly float _entryProcessorMinInvocationTime;
+
+ /** */
+ private readonly long _entryProcessorReadOnlyInvocations;
+
+ /** */
+ private readonly float _entryProcessorHitPercentage;
+
+ /** */
+ private readonly long _entryProcessorHits;
+
+ /** */
+ private readonly long _entryProcessorMisses;
+
+ /** */
+ private readonly float _entryProcessorMissPercentage;
+
+ /** */
+ private readonly long _entryProcessorRemovals;
+
/// <summary>
/// Initializes a new instance of the <see cref="CacheMetricsImpl"/> class.
/// </summary>
@@ -335,6 +368,17 @@ namespace Apache.Ignite.Core.Impl.Cache
_cacheSize = reader.ReadLong();
_rebalancedKeys = reader.ReadLong();
_estimatedRebalancedKeys = reader.ReadLong();
+ _entryProcessorPuts = reader.ReadLong();
+ _entryProcessorAverageInvocationTime = reader.ReadFloat();
+ _entryProcessorInvocations = reader.ReadLong();
+ _entryProcessorMaxInvocationTime = reader.ReadFloat();
+ _entryProcessorMinInvocationTime = reader.ReadFloat();
+ _entryProcessorReadOnlyInvocations = reader.ReadLong();
+ _entryProcessorHitPercentage = reader.ReadFloat();
+ _entryProcessorHits = reader.ReadLong();
+ _entryProcessorMisses = reader.ReadLong();
+ _entryProcessorMissPercentage = reader.ReadFloat();
+ _entryProcessorRemovals = reader.ReadLong();
}
/** <inheritDoc /> */
@@ -561,8 +605,41 @@ namespace Apache.Ignite.Core.Impl.Cache
/** <inheritDoc /> */
public long RebalancedKeys { get { return _rebalancedKeys; } }
-
+
/** <inheritDoc /> */
public long EstimatedRebalancingKeys { get { return _estimatedRebalancedKeys; } }
+
+ /** <inheritDoc /> */
+ public long EntryProcessorPuts { get { return _entryProcessorPuts; } }
+
+ /** <inheritDoc /> */
+ public float EntryProcessorAverageInvocationTime { get { return _entryProcessorAverageInvocationTime; } }
+
+ /** <inheritDoc /> */
+ public long EntryProcessorInvocations { get { return _entryProcessorInvocations; } }
+
+ /** <inheritDoc /> */
+ public float EntryProcessorMaxInvocationTime { get { return _entryProcessorMaxInvocationTime; } }
+
+ /** <inheritDoc /> */
+ public float EntryProcessorMinInvocationTime { get { return _entryProcessorMinInvocationTime; } }
+
+ /** <inheritDoc /> */
+ public long EntryProcessorReadOnlyInvocations { get { return _entryProcessorReadOnlyInvocations; } }
+
+ /** <inheritDoc /> */
+ public float EntryProcessorHitPercentage { get { return _entryProcessorHitPercentage; } }
+
+ /** <inheritDoc /> */
+ public long EntryProcessorHits { get { return _entryProcessorHits; } }
+
+ /** <inheritDoc /> */
+ public long EntryProcessorMisses { get { return _entryProcessorMisses; } }
+
+ /** <inheritDoc /> */
+ public float EntryProcessorMissPercentage { get { return _entryProcessorMissPercentage; } }
+
+ /** <inheritDoc /> */
+ public long EntryProcessorRemovals { get { return _entryProcessorRemovals; } }
}
}
\ No newline at end of file
[09/50] [abbrv] ignite git commit: IGNITE-7255 Web Console: Fixed
EXPLAIN execution on Queries screen.
Posted by av...@apache.org.
IGNITE-7255 Web Console: Fixed EXPLAIN execution on Queries screen.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d73211d2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d73211d2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d73211d2
Branch: refs/heads/ignite-8446
Commit: d73211d2c1fc3897681a3abdc98c5eb383d24475
Parents: c2383e9
Author: Vasiliy Sisko <vs...@gridgain.com>
Authored: Tue Jul 31 11:41:55 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Tue Jul 31 11:41:55 2018 +0700
----------------------------------------------------------------------
.../page-queries/components/queries-notebook/controller.js | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d73211d2/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
index 09c4184..132f7f6 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
@@ -1546,6 +1546,10 @@ export class NotebookCtrl {
};
$scope.explain = (paragraph) => {
+ const nonCollocatedJoins = !!paragraph.nonCollocatedJoins;
+ const enforceJoinOrder = !!paragraph.enforceJoinOrder;
+ const collocated = !!paragraph.collocated;
+
if (!$scope.queryAvailable(paragraph))
return;
@@ -1568,7 +1572,7 @@ export class NotebookCtrl {
ActivitiesData.post({ action: '/queries/explain' });
- return agentMgr.querySql(nid, args.cacheName, args.query, args.nonCollocatedJoins, !!paragraph.enforceJoinOrder, false, false, args.pageSize, false, args.collocated);
+ return agentMgr.querySql(nid, args.cacheName, args.query, nonCollocatedJoins, enforceJoinOrder, false, false, args.pageSize, false, collocated);
})
.then((res) => _processQueryResult(paragraph, true, res))
.catch((err) => {
[17/50] [abbrv] ignite git commit: IGNITE-9100 Split Basic and Cache
TC configurations on pure in-memory and with disk usage one
Posted by av...@apache.org.
IGNITE-9100 Split Basic and Cache TC configurations on pure in-memory and with disk usage one
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c5e723c5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c5e723c5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c5e723c5
Branch: refs/heads/ignite-8446
Commit: c5e723c533a336802ba0fa6b9e9f361182db16ff
Parents: 8309cef
Author: Eduard Shangareev <es...@gridgain.com>
Authored: Tue Jul 31 16:25:50 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Jul 31 16:25:50 2018 +0300
----------------------------------------------------------------------
.../testframework/junits/GridAbstractTest.java | 95 ++++++++++++++------
.../ignite/testsuites/IgniteBasicTestSuite.java | 12 ++-
.../IgniteBasicWithPersistenceTestSuite.java | 67 ++++++++++++++
...IgniteBinaryObjectsComputeGridTestSuite.java | 3 +
.../IgniteCacheEvictionSelfTestSuite.java | 3 -
.../IgniteCacheFullApiSelfTestSuite.java | 3 +
.../IgniteCacheMetricsSelfTestSuite.java | 4 -
.../ignite/testsuites/IgniteCacheTestSuite.java | 8 +-
.../testsuites/IgniteCacheTestSuite2.java | 5 +-
.../testsuites/IgniteCacheTestSuite3.java | 7 +-
.../testsuites/IgniteCacheTestSuite4.java | 5 +-
.../testsuites/IgniteCacheTestSuite5.java | 6 +-
.../testsuites/IgniteCacheTestSuite6.java | 28 +-----
.../testsuites/IgniteCacheTestSuite7.java | 53 ++++++++++-
.../testsuites/IgniteComputeGridTestSuite.java | 2 -
.../testsuites/IgniteKernalSelfTestSuite.java | 7 +-
.../IgniteMarshallerSelfTestSuite.java | 5 +-
.../testsuites/IgniteUtilSelfTestSuite.java | 7 +-
.../GridInternalTaskUnusedWalSegmentsTest.java | 27 +++---
...acheWithIndexingAndPersistenceTestSuite.java | 38 ++++++++
.../IgniteCacheWithIndexingTestSuite.java | 6 +-
21 files changed, 282 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index 1adb074..53ccdc7 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -17,31 +17,6 @@
package org.apache.ignite.testframework.junits;
-import java.io.ObjectStreamException;
-import java.io.Serializable;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.lang.reflect.Proxy;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import javax.cache.configuration.Factory;
-import javax.cache.configuration.FactoryBuilder;
import junit.framework.TestCase;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
@@ -55,6 +30,8 @@ import org.apache.ignite.binary.BinaryBasicNameMapper;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.BinaryConfiguration;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.NearCacheConfiguration;
import org.apache.ignite.events.EventType;
@@ -121,6 +98,32 @@ import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.FileSystemXmlApplicationContext;
+import javax.cache.configuration.Factory;
+import javax.cache.configuration.FactoryBuilder;
+import java.io.ObjectStreamException;
+import java.io.Serializable;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.Proxy;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
import static org.apache.ignite.IgniteSystemProperties.IGNITE_CLIENT_CACHE_CHANGE_MESSAGE_TIMEOUT;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
@@ -138,6 +141,9 @@ import static org.apache.ignite.testframework.config.GridTestProperties.IGNITE_C
"JUnitTestCaseWithNonTrivialConstructors"
})
public abstract class GridAbstractTest extends TestCase {
+ /** Persistence in tests allowed property. */
+ public static final String PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY = "PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY";
+
/**************************************************************
* DO NOT REMOVE TRANSIENT - THIS OBJECT MIGHT BE TRANSFERRED *
* TO ANOTHER NODE. *
@@ -198,6 +204,12 @@ public abstract class GridAbstractTest extends TestCase {
/**
*
*/
+ private static final boolean PERSISTENCE_ALLOWED =
+ IgniteSystemProperties.getBoolean(PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, true);
+
+ /**
+ *
+ */
static {
System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_ATOMIC_OPS_IN_TX, "false");
System.setProperty(IgniteSystemProperties.IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE, "10000");
@@ -856,6 +868,9 @@ public abstract class GridAbstractTest extends TestCase {
*/
protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg, GridSpringResourceContext ctx)
throws Exception {
+
+ checkConfiguration(cfg);
+
if (!isRemoteJvm(igniteInstanceName)) {
startingIgniteInstanceName.set(igniteInstanceName);
@@ -901,6 +916,30 @@ public abstract class GridAbstractTest extends TestCase {
}
/**
+ * @param cfg Config.
+ */
+ protected void checkConfiguration(IgniteConfiguration cfg) {
+ if (cfg == null)
+ return;
+
+ if (!PERSISTENCE_ALLOWED) {
+ String errorMsg = "PERSISTENCE IS NOT ALLOWED IN THIS SUITE, PUT YOUR TEST TO ANOTHER ONE!";
+
+ DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration();
+
+ if (dsCfg != null) {
+ assertFalse(errorMsg, dsCfg.getDefaultDataRegionConfiguration().isPersistenceEnabled());
+
+ DataRegionConfiguration[] dataRegionConfigurations = dsCfg.getDataRegionConfigurations();
+
+ if (dataRegionConfigurations != null)
+ for (DataRegionConfiguration dataRegionConfiguration : dataRegionConfigurations)
+ assertFalse(errorMsg, dataRegionConfiguration.isPersistenceEnabled());
+ }
+ }
+ }
+
+ /**
* Starts new grid at another JVM with given name.
*
* @param igniteInstanceName Ignite instance name.
@@ -996,6 +1035,8 @@ public abstract class GridAbstractTest extends TestCase {
if (cfg == null)
cfg = optimize(getConfiguration(igniteInstanceName));
+ checkConfiguration(cfg);
+
if (locNode != null) {
DiscoverySpi discoverySpi = locNode.configuration().getDiscoverySpi();
@@ -1034,6 +1075,8 @@ public abstract class GridAbstractTest extends TestCase {
* @throws IgniteCheckedException On error.
*/
protected IgniteConfiguration optimize(IgniteConfiguration cfg) throws IgniteCheckedException {
+ checkConfiguration(cfg);
+
if (cfg.getLocalHost() == null) {
if (cfg.getDiscoverySpi() instanceof TcpDiscoverySpi) {
cfg.setLocalHost("127.0.0.1");
@@ -1635,6 +1678,8 @@ public abstract class GridAbstractTest extends TestCase {
cfg.setFailureHandler(getFailureHandler(igniteInstanceName));
+ checkConfiguration(cfg);
+
return cfg;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index c75ab7d..74e23ed 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -17,16 +17,12 @@
package org.apache.ignite.testsuites;
-import java.util.Set;
import junit.framework.TestSuite;
import org.apache.ignite.GridSuppressedExceptionSelfTest;
-import org.apache.ignite.failure.AccountTransferTransactionTest;
import org.apache.ignite.failure.FailureHandlerTriggeredTest;
-import org.apache.ignite.failure.IoomFailureHandlerTest;
import org.apache.ignite.failure.OomFailureHandlerTest;
import org.apache.ignite.failure.StopNodeFailureHandlerTest;
import org.apache.ignite.failure.StopNodeOrHaltFailureHandlerTest;
-import org.apache.ignite.failure.SystemWorkersTerminationTest;
import org.apache.ignite.internal.ClassSetTest;
import org.apache.ignite.internal.ClusterGroupHostsSelfTest;
import org.apache.ignite.internal.ClusterGroupSelfTest;
@@ -87,12 +83,15 @@ import org.apache.ignite.plugin.security.SecurityPermissionSetBuilderTest;
import org.apache.ignite.spi.GridSpiLocalHostInjectionTest;
import org.apache.ignite.startup.properties.NotStringSystemPropertyTest;
import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
import org.apache.ignite.testframework.test.ConfigVariationsTestSuiteBuilderTest;
import org.apache.ignite.testframework.test.ParametersTest;
import org.apache.ignite.testframework.test.VariationsIteratorTest;
import org.apache.ignite.util.AttributeNodeFilterSelfTest;
import org.jetbrains.annotations.Nullable;
+import java.util.Set;
+
/**
* Basic test suite.
*/
@@ -111,6 +110,8 @@ public class IgniteBasicTestSuite extends TestSuite {
* @throws Exception Thrown in case of the failure.
*/
public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Exception {
+ System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
TestSuite suite = new TestSuite("Ignite Basic Test Suite");
suite.addTest(IgniteMarshallerSelfTestSuite.suite(ignoredTests));
@@ -209,10 +210,7 @@ public class IgniteBasicTestSuite extends TestSuite {
suite.addTestSuite(FailureHandlerTriggeredTest.class);
suite.addTestSuite(StopNodeFailureHandlerTest.class);
suite.addTestSuite(StopNodeOrHaltFailureHandlerTest.class);
- suite.addTestSuite(IoomFailureHandlerTest.class);
suite.addTestSuite(OomFailureHandlerTest.class);
- suite.addTestSuite(AccountTransferTransactionTest.class);
- suite.addTestSuite(SystemWorkersTerminationTest.class);
suite.addTestSuite(AtomicOperationsInTxTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
new file mode 100644
index 0000000..5f1d18d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.failure.AccountTransferTransactionTest;
+import org.apache.ignite.failure.IoomFailureHandlerTest;
+import org.apache.ignite.failure.SystemWorkersTerminationTest;
+import org.apache.ignite.internal.ClusterBaselineNodesMetricsSelfTest;
+import org.apache.ignite.internal.processors.service.ServiceDeploymentOnActivationTest;
+import org.apache.ignite.internal.processors.service.ServiceDeploymentOutsideBaselineTest;
+import org.apache.ignite.marshaller.GridMarshallerMappingConsistencyTest;
+import org.apache.ignite.util.GridCommandHandlerTest;
+import org.apache.ignite.util.GridInternalTaskUnusedWalSegmentsTest;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Set;
+
+/**
+ * Basic test suite.
+ */
+public class IgniteBasicWithPersistenceTestSuite extends TestSuite {
+ /**
+ * @return Test suite.
+ * @throws Exception Thrown in case of the failure.
+ */
+ public static TestSuite suite() throws Exception {
+ return suite(null);
+ }
+
+ /**
+ * @param ignoredTests Tests don't include in the execution. Providing null means nothing to exclude.
+ * @return Test suite.
+ * @throws Exception Thrown in case of the failure.
+ */
+ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Exception {
+ TestSuite suite = new TestSuite("Ignite Basic With Persistence Test Suite");
+
+ suite.addTestSuite(IoomFailureHandlerTest.class);
+ suite.addTestSuite(AccountTransferTransactionTest.class);
+ suite.addTestSuite(ClusterBaselineNodesMetricsSelfTest.class);
+ suite.addTestSuite(ServiceDeploymentOnActivationTest.class);
+ suite.addTestSuite(ServiceDeploymentOutsideBaselineTest.class);
+ suite.addTestSuite(GridMarshallerMappingConsistencyTest.class);
+ suite.addTestSuite(SystemWorkersTerminationTest.class);
+
+ suite.addTestSuite(GridCommandHandlerTest.class);
+ suite.addTestSuite(GridInternalTaskUnusedWalSegmentsTest.class);
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java
index 8798db9..4820d45 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java
@@ -19,6 +19,7 @@ package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.internal.GridComputationBinarylizableClosuresSelfTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
/**
*
@@ -29,6 +30,8 @@ public class IgniteBinaryObjectsComputeGridTestSuite {
* @throws Exception If failed.
*/
public static TestSuite suite() throws Exception {
+ System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
TestSuite suite = IgniteComputeGridTestSuite.suite();
suite.addTestSuite(GridComputationBinarylizableClosuresSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java
index f2ac1c0..79d98e4 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java
@@ -37,7 +37,6 @@ import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearEvictionP
import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearOnlyNearEvictionPolicySelfTest;
import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionDataStreamerTest;
import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMetricTest;
-import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMultinodeMixedRegionsTest;
import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionPagesRecyclingAndReusingTest;
import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionReadThroughTest;
import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionTouchOrderTest;
@@ -91,8 +90,6 @@ public class IgniteCacheEvictionSelfTestSuite extends TestSuite {
suite.addTest(new TestSuite(PageEvictionReadThroughTest.class));
suite.addTest(new TestSuite(PageEvictionDataStreamerTest.class));
- suite.addTest(new TestSuite(PageEvictionMultinodeMixedRegionsTest.class));
-
suite.addTest(new TestSuite(PageEvictionMetricTest.class));
suite.addTest(new TestSuite(PageEvictionPagesRecyclingAndReusingTest.class));
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
index b380ebc..bc46055 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
@@ -78,6 +78,7 @@ import org.apache.ignite.internal.processors.cache.local.GridCacheLocalAtomicWit
import org.apache.ignite.internal.processors.cache.local.GridCacheLocalFullApiMultithreadedSelfTest;
import org.apache.ignite.internal.processors.cache.local.GridCacheLocalFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.local.GridCacheLocalWithGroupFullApiSelfTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
/**
* Test suite for cache API.
@@ -88,6 +89,8 @@ public class IgniteCacheFullApiSelfTestSuite extends TestSuite {
* @throws Exception If failed.
*/
public static TestSuite suite() throws Exception {
+ System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
TestSuite suite = new TestSuite("Cache Full API Test Suite");
// One node.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
index 6f62ffb..b6dcb21 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
@@ -19,9 +19,7 @@ package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.internal.TransactionMetricsMxBeanImplTest;
-import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMBeanTest;
import org.apache.ignite.internal.processors.cache.CacheGroupsMetricsRebalanceTest;
-import org.apache.ignite.internal.processors.cache.CacheMetricsManageTest;
import org.apache.ignite.internal.processors.cache.CacheMetricsEntitiesCountTest;
import org.apache.ignite.internal.processors.cache.CacheMetricsForClusterGroupSelfTest;
import org.apache.ignite.internal.processors.cache.CacheValidatorMetricsTest;
@@ -69,9 +67,7 @@ public class IgniteCacheMetricsSelfTestSuite extends TestSuite {
suite.addTestSuite(GridCacheAtomicLocalTckMetricsSelfTestImpl.class);
suite.addTestSuite(CacheGroupsMetricsRebalanceTest.class);
- suite.addTestSuite(CacheGroupMetricsMBeanTest.class);
suite.addTestSuite(CacheValidatorMetricsTest.class);
- suite.addTestSuite(CacheMetricsManageTest.class);
suite.addTestSuite(CacheMetricsEntitiesCountTest.class);
// Cluster wide metrics.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index c607284..f8a378f 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -17,7 +17,6 @@
package org.apache.ignite.testsuites;
-import java.util.Set;
import junit.framework.TestSuite;
import org.apache.ignite.cache.IgniteCacheEntryProcessorSequentialCallTest;
import org.apache.ignite.cache.IgniteWarmupClosureSelfTest;
@@ -52,6 +51,7 @@ import org.apache.ignite.internal.processors.cache.CacheNamesSelfTest;
import org.apache.ignite.internal.processors.cache.CacheNamesWithSpecialCharactersTest;
import org.apache.ignite.internal.processors.cache.CachePutEventListenerErrorSelfTest;
import org.apache.ignite.internal.processors.cache.CacheTxFastFinishTest;
+import org.apache.ignite.internal.processors.cache.DataStorageConfigurationValidationTest;
import org.apache.ignite.internal.processors.cache.GridCacheAffinityApiSelfTest;
import org.apache.ignite.internal.processors.cache.GridCacheAffinityMapperSelfTest;
import org.apache.ignite.internal.processors.cache.GridCacheAffinityRoutingSelfTest;
@@ -83,7 +83,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheSwapPreloadSelfTest;
import org.apache.ignite.internal.processors.cache.GridCacheTtlManagerSelfTest;
import org.apache.ignite.internal.processors.cache.GridCacheTxPartitionedLocalStoreSelfTest;
import org.apache.ignite.internal.processors.cache.GridDataStorageConfigurationConsistencySelfTest;
-import org.apache.ignite.internal.processors.cache.DataStorageConfigurationValidationTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicInvokeTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalInvokeTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalWithStoreInvokeTest;
@@ -150,6 +149,9 @@ import org.apache.ignite.internal.processors.datastreamer.DataStreamerMultinodeC
import org.apache.ignite.internal.processors.datastreamer.DataStreamerTimeoutTest;
import org.apache.ignite.internal.processors.datastreamer.DataStreamerUpdateAfterLoadTest;
import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
+
+import java.util.Set;
/**
* Test suite.
@@ -169,6 +171,8 @@ public class IgniteCacheTestSuite extends TestSuite {
* @throws Exception Thrown in case of the failure.
*/
public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
+ System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
TestSuite suite = new TestSuite("IgniteCache Test Suite");
suite.addTestSuite(IgniteCacheEntryListenerAtomicTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
index e9c5e76..6eca728 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
@@ -31,7 +31,6 @@ import org.apache.ignite.internal.processors.cache.CacheEnumOperationsSingleNode
import org.apache.ignite.internal.processors.cache.CacheEnumOperationsTest;
import org.apache.ignite.internal.processors.cache.CacheExchangeMessageDuplicatedStateTest;
import org.apache.ignite.internal.processors.cache.CacheGroupLocalConfigurationSelfTest;
-import org.apache.ignite.internal.processors.cache.CacheDataRegionConfigurationTest;
import org.apache.ignite.internal.processors.cache.CacheOptimisticTransactionsWithFilterSingleServerTest;
import org.apache.ignite.internal.processors.cache.CacheOptimisticTransactionsWithFilterTest;
import org.apache.ignite.internal.processors.cache.CrossCacheTxNearEnabledRandomOperationsTest;
@@ -144,6 +143,7 @@ import org.apache.ignite.internal.processors.cache.local.GridCacheLocalTxSingleT
import org.apache.ignite.internal.processors.cache.local.GridCacheLocalTxTimeoutSelfTest;
import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicyInitializationTest;
import org.apache.ignite.internal.processors.continuous.IgniteNoCustomEventsOnNodeStart;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
/**
* Test suite.
@@ -154,6 +154,8 @@ public class IgniteCacheTestSuite2 extends TestSuite {
* @throws Exception Thrown in case of the failure.
*/
public static TestSuite suite() throws Exception {
+ System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
TestSuite suite = new TestSuite("IgniteCache Test Suite part 2");
// Local cache.
@@ -266,7 +268,6 @@ public class IgniteCacheTestSuite2 extends TestSuite {
suite.addTest(new TestSuite(CacheConfigurationLeakTest.class));
suite.addTest(new TestSuite(MemoryPolicyConfigValidationTest.class));
suite.addTest(new TestSuite(MemoryPolicyInitializationTest.class));
- suite.addTest(new TestSuite(CacheDataRegionConfigurationTest.class));
suite.addTest(new TestSuite(CacheGroupLocalConfigurationSelfTest.class));
suite.addTest(new TestSuite(CacheEnumOperationsSingleNodeTest.class));
suite.addTest(new TestSuite(CacheEnumOperationsTest.class));
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index b66bf5b..55436d6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -53,11 +53,9 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePut
import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxReentryNearSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRabalancingDelayedPartitionMapExchangeSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingAsyncSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheDaemonNodeReplicatedSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicGetAndTransformStoreSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedBasicApiTest;
@@ -84,6 +82,7 @@ import org.apache.ignite.internal.processors.cache.distributed.replicated.preloa
import org.apache.ignite.internal.processors.cache.distributed.replicated.preloader.GridCacheReplicatedPreloadStartStopEventsSelfTest;
import org.apache.ignite.internal.processors.cache.local.GridCacheDaemonNodeLocalSelfTest;
import org.apache.ignite.internal.processors.cache.local.GridCacheLocalByteArrayValuesSelfTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
/**
* Test suite.
@@ -94,6 +93,8 @@ public class IgniteCacheTestSuite3 extends TestSuite {
* @throws Exception Thrown in case of the failure.
*/
public static TestSuite suite() throws Exception {
+ System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
TestSuite suite = new TestSuite("IgniteCache Test Suite part 3");
suite.addTestSuite(IgniteCacheGroupsTest.class);
@@ -151,8 +152,6 @@ public class IgniteCacheTestSuite3 extends TestSuite {
suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class);
suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
- suite.addTestSuite(GridCacheRebalancingPartitionCountersTest.class);
- suite.addTestSuite(GridCacheRebalancingWithAsyncClearingTest.class);
// Test for byte array value special case.
suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index b973c91..321046b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -80,7 +80,6 @@ import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheFilterTest;
import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheMultinodeTest;
import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartCoordinatorFailoverTest;
import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartFailTest;
-import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartFailWithPersistenceTest;
import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartNoExchangeTimeoutTest;
import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartSelfTest;
import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartStopConcurrentTest;
@@ -151,6 +150,7 @@ import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryPa
import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryPartitionedTransactionalSelfTest;
import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryReplicatedAtomicSelfTest;
import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryReplicatedTransactionalSelfTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
/**
* Test suite.
@@ -161,6 +161,8 @@ public class IgniteCacheTestSuite4 extends TestSuite {
* @throws Exception Thrown in case of the failure.
*/
public static TestSuite suite() throws Exception {
+ System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
TestSuite suite = new TestSuite("IgniteCache Test Suite part 4");
// Multi node update.
@@ -229,7 +231,6 @@ public class IgniteCacheTestSuite4 extends TestSuite {
suite.addTestSuite(IgniteDynamicCacheStartSelfTest.class);
suite.addTestSuite(IgniteDynamicCacheMultinodeTest.class);
suite.addTestSuite(IgniteDynamicCacheStartFailTest.class);
- suite.addTestSuite(IgniteDynamicCacheStartFailWithPersistenceTest.class);
suite.addTestSuite(IgniteDynamicCacheStartCoordinatorFailoverTest.class);
suite.addTestSuite(IgniteDynamicCacheWithConfigStartSelfTest.class);
suite.addTestSuite(IgniteCacheDynamicStopSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 945a76c..1c74d6d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -37,7 +37,6 @@ import org.apache.ignite.internal.processors.cache.IgniteCachePutStackOverflowSe
import org.apache.ignite.internal.processors.cache.IgniteCacheReadThroughEvictionsVariationsSuite;
import org.apache.ignite.internal.processors.cache.IgniteCacheStoreCollectionTest;
import org.apache.ignite.internal.processors.cache.PartitionsExchangeOnDiscoveryHistoryOverflowTest;
-import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentTest;
import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheGroupsPartitionLossPolicySelfTest;
@@ -48,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.Ignite
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.CacheManualRebalancingTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest;
import org.apache.ignite.internal.processors.cache.store.IgniteCacheWriteBehindNoUpdateSelfTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
/**
* Test suite.
@@ -58,6 +58,8 @@ public class IgniteCacheTestSuite5 extends TestSuite {
* @throws Exception Thrown in case of the failure.
*/
public static TestSuite suite() throws Exception {
+ System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
TestSuite suite = new TestSuite("IgniteCache Test Suite part 5");
suite.addTestSuite(CacheSerializableTransactionsTest.class);
@@ -102,8 +104,6 @@ public class IgniteCacheTestSuite5 extends TestSuite {
suite.addTestSuite(ConcurrentCacheStartTest.class);
- suite.addTestSuite(Cache64kPartitionsTest.class);
-
return suite;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index 77285be..7a5aa80 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -18,20 +18,12 @@
package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
-import org.apache.ignite.internal.processors.authentication.AuthenticationConfigurationClusterTest;
-import org.apache.ignite.internal.processors.authentication.AuthenticationOnNotActiveClusterTest;
-import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorNPEOnStartTest;
-import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorNodeRestartTest;
-import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorSelfTest;
import org.apache.ignite.internal.processors.cache.PartitionedAtomicCacheGetsDistributionTest;
import org.apache.ignite.internal.processors.cache.PartitionedTransactionalOptimisticCacheGetsDistributionTest;
import org.apache.ignite.internal.processors.cache.PartitionedTransactionalPessimisticCacheGetsDistributionTest;
import org.apache.ignite.internal.processors.cache.ReplicatedAtomicCacheGetsDistributionTest;
import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalOptimisticCacheGetsDistributionTest;
import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalPessimisticCacheGetsDistributionTest;
-import org.apache.ignite.internal.processors.cache.WalModeChangeAdvancedSelfTest;
-import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAffinityNodeSelfTest;
-import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
import org.apache.ignite.internal.processors.cache.datastructures.IgniteExchangeLatchManagerCoordinatorFailTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheExchangeMergeTest;
import org.apache.ignite.internal.processors.cache.distributed.CachePartitionStateTest;
@@ -41,7 +33,6 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheThread
import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeMultiServerTest;
import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeTest;
import org.apache.ignite.internal.processors.cache.distributed.IgnitePessimisticTxSuspendResumeTest;
-import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
import org.apache.ignite.internal.processors.cache.transactions.TxLabelTest;
import org.apache.ignite.internal.processors.cache.transactions.TxMultiCacheAsyncOpsTest;
import org.apache.ignite.internal.processors.cache.transactions.TxOnCachesStartTest;
@@ -49,12 +40,11 @@ import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticOnPa
import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticPrepareOnUnstableTopologyTest;
import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncNearCacheTest;
import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncTest;
-import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncWithPersistenceTest;
import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNearCacheTest;
import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNoDeadlockDetectionTest;
import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest;
import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTopologyChangeTest;
-import org.apache.ignite.internal.processors.cache.transactions.TxWithSmallTimeoutAndContentionOneKeyTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
/**
* Test suite.
@@ -65,6 +55,8 @@ public class IgniteCacheTestSuite6 extends TestSuite {
* @throws Exception Thrown in case of the failure.
*/
public static TestSuite suite() throws Exception {
+ System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
TestSuite suite = new TestSuite("IgniteCache Test Suite part 6");
suite.addTestSuite(CachePartitionStateTest.class);
@@ -79,11 +71,9 @@ public class IgniteCacheTestSuite6 extends TestSuite {
suite.addTestSuite(TxRollbackOnTimeoutTest.class);
suite.addTestSuite(TxRollbackOnTimeoutNoDeadlockDetectionTest.class);
suite.addTestSuite(TxRollbackOnTimeoutNearCacheTest.class);
- suite.addTestSuite(TxWithSmallTimeoutAndContentionOneKeyTest.class);
suite.addTestSuite(IgniteCacheThreadLocalTxTest.class);
suite.addTestSuite(TxRollbackAsyncTest.class);
suite.addTestSuite(TxRollbackAsyncNearCacheTest.class);
- suite.addTestSuite(TxRollbackAsyncWithPersistenceTest.class);
suite.addTestSuite(TxRollbackOnTopologyChangeTest.class);
suite.addTestSuite(TxOptimisticPrepareOnUnstableTopologyTest.class);
@@ -94,23 +84,11 @@ public class IgniteCacheTestSuite6 extends TestSuite {
suite.addTestSuite(TxOnCachesStartTest.class);
- suite.addTestSuite(IgnitePdsCacheAssignmentNodeRestartsTest.class);
-
- suite.addTestSuite(WalModeChangeSelfTest.class);
- suite.addTestSuite(WalModeChangeCoordinatorNotAffinityNodeSelfTest.class);
- suite.addTestSuite(WalModeChangeAdvancedSelfTest.class);
-
suite.addTestSuite(IgniteCache150ClientsTest.class);
// TODO enable this test after IGNITE-6753, now it takes too long
// suite.addTestSuite(IgniteOutOfMemoryPropagationTest.class);
- suite.addTestSuite(AuthenticationConfigurationClusterTest.class);
- suite.addTestSuite(AuthenticationProcessorSelfTest.class);
- suite.addTestSuite(AuthenticationOnNotActiveClusterTest.class);
- suite.addTestSuite(AuthenticationProcessorNodeRestartTest.class);
- suite.addTestSuite(AuthenticationProcessorNPEOnStartTest.class);
-
suite.addTestSuite(ReplicatedAtomicCacheGetsDistributionTest.class);
suite.addTestSuite(ReplicatedTransactionalOptimisticCacheGetsDistributionTest.class);
suite.addTestSuite(ReplicatedTransactionalPessimisticCacheGetsDistributionTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
index dbd455a..330796c 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
@@ -17,9 +17,30 @@
package org.apache.ignite.testsuites;
-import java.util.Set;
import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.authentication.Authentication1kUsersNodeRestartTest;
+import org.apache.ignite.internal.processors.authentication.AuthenticationConfigurationClusterTest;
+import org.apache.ignite.internal.processors.authentication.AuthenticationOnNotActiveClusterTest;
+import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorNPEOnStartTest;
+import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorNodeRestartTest;
+import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorSelfTest;
+import org.apache.ignite.internal.processors.cache.CacheDataRegionConfigurationTest;
+import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMBeanTest;
+import org.apache.ignite.internal.processors.cache.CacheMetricsManageTest;
+import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartFailWithPersistenceTest;
+import org.apache.ignite.internal.processors.cache.WalModeChangeAdvancedSelfTest;
+import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAffinityNodeSelfTest;
+import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingTest;
+import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMultinodeMixedRegionsTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
import org.apache.ignite.internal.processors.cache.persistence.db.CheckpointBufferDeadlockTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncWithPersistenceTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxWithSmallTimeoutAndContentionOneKeyTest;
+
+import java.util.Set;
/**
* Test suite.
@@ -39,10 +60,38 @@ public class IgniteCacheTestSuite7 extends TestSuite {
* @throws Exception Thrown in case of the failure.
*/
public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
- TestSuite suite = new TestSuite("IgniteCache Test Suite part 7");
+ TestSuite suite = new TestSuite("IgniteCache With Persistence Test Suite");
suite.addTestSuite(CheckpointBufferDeadlockTest.class);
+ suite.addTestSuite(AuthenticationConfigurationClusterTest.class);
+ suite.addTestSuite(AuthenticationProcessorSelfTest.class);
+ suite.addTestSuite(AuthenticationOnNotActiveClusterTest.class);
+ suite.addTestSuite(AuthenticationProcessorNodeRestartTest.class);
+ suite.addTestSuite(AuthenticationProcessorNPEOnStartTest.class);
+ suite.addTestSuite(Authentication1kUsersNodeRestartTest.class);
+
+ suite.addTestSuite(CacheDataRegionConfigurationTest.class);
+
+ suite.addTestSuite(WalModeChangeAdvancedSelfTest.class);
+ suite.addTestSuite(WalModeChangeSelfTest.class);
+ suite.addTestSuite(WalModeChangeCoordinatorNotAffinityNodeSelfTest.class);
+
+ suite.addTestSuite(Cache64kPartitionsTest.class);
+ suite.addTestSuite(GridCacheRebalancingPartitionCountersTest.class);
+ suite.addTestSuite(GridCacheRebalancingWithAsyncClearingTest.class);
+
+ suite.addTestSuite(IgnitePdsCacheAssignmentNodeRestartsTest.class);
+ suite.addTestSuite(TxRollbackAsyncWithPersistenceTest.class);
+
+ suite.addTestSuite(CacheGroupMetricsMBeanTest.class);
+ suite.addTestSuite(CacheMetricsManageTest.class);
+ suite.addTestSuite(PageEvictionMultinodeMixedRegionsTest.class);
+
+ suite.addTestSuite(IgniteDynamicCacheStartFailWithPersistenceTest.class);
+
+ suite.addTestSuite(TxWithSmallTimeoutAndContentionOneKeyTest.class);
+
return suite;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index bab7099..14eb296 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -18,7 +18,6 @@
package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
-import org.apache.ignite.internal.ClusterBaselineNodesMetricsSelfTest;
import org.apache.ignite.internal.ClusterNodeMetricsSelfTest;
import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest;
import org.apache.ignite.internal.GridAffinityNoCacheSelfTest;
@@ -125,7 +124,6 @@ public class IgniteComputeGridTestSuite {
suite.addTestSuite(GridTaskInstanceExecutionSelfTest.class);
suite.addTestSuite(ClusterNodeMetricsSelfTest.class);
suite.addTestSuite(ClusterNodeMetricsUpdateTest.class);
- suite.addTestSuite(ClusterBaselineNodesMetricsSelfTest.class);
suite.addTestSuite(GridNonHistoryMetricsSelfTest.class);
suite.addTestSuite(GridCancelledJobsMetricsSelfTest.class);
suite.addTestSuite(GridCollisionJobsContextSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index 0b7d2d2..af9e151 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -17,7 +17,6 @@
package org.apache.ignite.testsuites;
-import java.util.Set;
import junit.framework.TestSuite;
import org.apache.ignite.internal.ComputeJobCancelWithServiceSelfTest;
import org.apache.ignite.internal.GridCommunicationSelfTest;
@@ -70,8 +69,6 @@ import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClas
import org.apache.ignite.internal.processors.service.IgniteServiceDynamicCachesSelfTest;
import org.apache.ignite.internal.processors.service.IgniteServiceProxyTimeoutInitializedTest;
import org.apache.ignite.internal.processors.service.IgniteServiceReassignmentTest;
-import org.apache.ignite.internal.processors.service.ServiceDeploymentOnActivationTest;
-import org.apache.ignite.internal.processors.service.ServiceDeploymentOutsideBaselineTest;
import org.apache.ignite.internal.processors.service.ServicePredicateAccessCacheTest;
import org.apache.ignite.internal.processors.service.SystemCacheNotConfiguredTest;
import org.apache.ignite.internal.util.GridStartupWithUndefinedIgniteHomeSelfTest;
@@ -79,6 +76,8 @@ import org.apache.ignite.services.ServiceThreadPoolSelfTest;
import org.apache.ignite.spi.communication.GridCacheMessageSelfTest;
import org.apache.ignite.testframework.GridTestUtils;
+import java.util.Set;
+
/**
* Kernal self test suite.
*/
@@ -153,8 +152,6 @@ public class IgniteKernalSelfTestSuite extends TestSuite {
suite.addTestSuite(ServiceThreadPoolSelfTest.class);
suite.addTestSuite(GridServiceProcessorBatchDeploySelfTest.class);
suite.addTestSuite(GridServiceDeploymentCompoundFutureSelfTest.class);
- suite.addTestSuite(ServiceDeploymentOnActivationTest.class);
- suite.addTestSuite(ServiceDeploymentOutsideBaselineTest.class);
suite.addTestSuite(SystemCacheNotConfiguredTest.class);
suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
index 0eb50d7..f5e3809 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
@@ -17,7 +17,6 @@
package org.apache.ignite.testsuites;
-import java.util.Set;
import junit.framework.TestSuite;
import org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2ByteOrderSelfTest;
import org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerEnumSelfTest;
@@ -30,11 +29,12 @@ import org.apache.ignite.internal.marshaller.optimized.OptimizedObjectStreamSelf
import org.apache.ignite.internal.util.GridHandleTableSelfTest;
import org.apache.ignite.internal.util.io.GridUnsafeDataInputOutputByteOrderSelfTest;
import org.apache.ignite.internal.util.io.GridUnsafeDataOutputArraySizingSelfTest;
-import org.apache.ignite.marshaller.GridMarshallerMappingConsistencyTest;
import org.apache.ignite.marshaller.MarshallerEnumDeadlockMultiJvmTest;
import org.apache.ignite.marshaller.jdk.GridJdkMarshallerSelfTest;
import org.apache.ignite.testframework.GridTestUtils;
+import java.util.Set;
+
/**
* Test suite for all marshallers.
*/
@@ -67,7 +67,6 @@ public class IgniteMarshallerSelfTestSuite extends TestSuite {
GridTestUtils.addTestIfNeeded(suite, DirectByteBufferStreamImplV2ByteOrderSelfTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, GridHandleTableSelfTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, OptimizedMarshallerPooledSelfTest.class, ignoredTests);
- GridTestUtils.addTestIfNeeded(suite, GridMarshallerMappingConsistencyTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, MarshallerEnumDeadlockMultiJvmTest.class, ignoredTests);
return suite;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index 791621f..309af8c 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -17,7 +17,6 @@
package org.apache.ignite.testsuites;
-import java.util.Set;
import junit.framework.TestSuite;
import org.apache.ignite.internal.commandline.CommandHandlerParsingTest;
import org.apache.ignite.internal.pagemem.impl.PageIdUtilsSelfTest;
@@ -41,9 +40,7 @@ import org.apache.ignite.spi.discovery.ClusterMetricsSnapshotSerializeSelfTest;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.thread.GridThreadPoolExecutorServiceSelfTest;
import org.apache.ignite.thread.IgniteThreadPoolSizeTest;
-import org.apache.ignite.util.GridCommandHandlerTest;
import org.apache.ignite.util.GridIntListSelfTest;
-import org.apache.ignite.util.GridInternalTaskUnusedWalSegmentsTest;
import org.apache.ignite.util.GridLongListSelfTest;
import org.apache.ignite.util.GridMessageCollectionTest;
import org.apache.ignite.util.GridPartitionMapSelfTest;
@@ -55,6 +52,8 @@ import org.apache.ignite.util.mbeans.GridMBeanExoticNamesSelfTest;
import org.apache.ignite.util.mbeans.GridMBeanSelfTest;
import org.apache.ignite.util.mbeans.WorkersControlMXBeanTest;
+import java.util.Set;
+
/**
* Test suite for Ignite utility classes.
*/
@@ -118,8 +117,6 @@ public class IgniteUtilSelfTestSuite extends TestSuite {
// control.sh
suite.addTestSuite(CommandHandlerParsingTest.class);
- suite.addTestSuite(GridCommandHandlerTest.class);
- suite.addTestSuite(GridInternalTaskUnusedWalSegmentsTest.class);
return suite;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java
index 148ecef..a39b0c2 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java
@@ -17,12 +17,7 @@
package org.apache.ignite.util;
-import java.io.File;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataRegionConfiguration;
@@ -40,6 +35,12 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import java.io.File;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
/**
@@ -101,15 +102,17 @@ public class GridInternalTaskUnusedWalSegmentsTest extends GridCommonAbstractTes
ig0.cluster().active(true);
- IgniteCache<Object, Object> cache = ig0.cache(DEFAULT_CACHE_NAME);
-
- for (int k = 0; k < 10_000; k++)
- cache.put(k, new byte[1024]);
+ try (IgniteDataStreamer streamer = ig0.dataStreamer(DEFAULT_CACHE_NAME)) {
+ for (int k = 0; k < 10_000; k++)
+ streamer.addData(k, new byte[1024]);
+ }
forceCheckpoint();
- for (int k = 0; k < 1_000; k++)
- cache.put(k, new byte[1024]);
+ try (IgniteDataStreamer streamer = ig0.dataStreamer(DEFAULT_CACHE_NAME)) {
+ for (int k = 0; k < 1_000; k++)
+ streamer.addData(k, new byte[1024]);
+ }
forceCheckpoint();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java
new file mode 100644
index 0000000..1bc60f6
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.util.GridCommandHandlerIndexingTest;
+
+/**
+ * Cache tests using indexing.
+ */
+public class IgniteCacheWithIndexingAndPersistenceTestSuite extends TestSuite {
+ /**
+ * @return Test suite.
+ * @throws Exception Thrown in case of the failure.
+ */
+ public static TestSuite suite() throws Exception {
+ TestSuite suite = new TestSuite("Ignite Cache With Indexing And Persistence Test Suite");
+
+ suite.addTestSuite(GridCommandHandlerIndexingTest.class);
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c5e723c5/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
index 0a9d8af..e351cb6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
@@ -41,7 +41,7 @@ import org.apache.ignite.internal.processors.cache.ttl.CacheTtlTransactionalLoca
import org.apache.ignite.internal.processors.cache.ttl.CacheTtlTransactionalPartitionedSelfTest;
import org.apache.ignite.internal.processors.client.IgniteDataStreamerTest;
import org.apache.ignite.internal.processors.query.h2.database.InlineIndexHelperTest;
-import org.apache.ignite.util.GridCommandHandlerIndexingTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
/**
* Cache tests using indexing.
@@ -52,6 +52,8 @@ public class IgniteCacheWithIndexingTestSuite extends TestSuite {
* @throws Exception Thrown in case of the failure.
*/
public static TestSuite suite() throws Exception {
+ System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
TestSuite suite = new TestSuite("Ignite Cache With Indexing Test Suite");
suite.addTestSuite(InlineIndexHelperTest.class);
@@ -86,8 +88,6 @@ public class IgniteCacheWithIndexingTestSuite extends TestSuite {
suite.addTestSuite(IgniteDataStreamerTest.class);
- suite.addTestSuite(GridCommandHandlerIndexingTest.class);
-
suite.addTestSuite(BinaryTypeMismatchLoggingTest.class);
return suite;
[26/50] [abbrv] ignite git commit: IGNITE-8799 Web Agent: Added
output to log of root cause if connection failed.
Posted by av...@apache.org.
IGNITE-8799 Web Agent: Added output to log of root cause if connection failed.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6cc633b7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6cc633b7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6cc633b7
Branch: refs/heads/ignite-8446
Commit: 6cc633b7dae8f7252ea30a75c4b2cacc8f3fe6fb
Parents: 19f9196
Author: Mikhail Cherkasov <mi...@gmail.com>
Authored: Wed Aug 1 12:35:02 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Aug 1 12:35:02 2018 +0700
----------------------------------------------------------------------
.../java/org/apache/ignite/console/agent/AgentLauncher.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6cc633b7/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
----------------------------------------------------------------------
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
index 9340417..579f236 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
@@ -114,7 +114,7 @@ public class AgentLauncher {
Exception ignore = X.cause(e, SSLHandshakeException.class);
if (ignore != null) {
- log.error("Failed to establish SSL connection to server, due to errors with SSL handshake.");
+ log.error("Failed to establish SSL connection to server, due to errors with SSL handshake:", e);
log.error("Add to environment variable JVM_OPTS parameter \"-Dtrust.all=true\" to skip certificate validation in case of using self-signed certificate.");
System.exit(1);
@@ -123,7 +123,7 @@ public class AgentLauncher {
ignore = X.cause(e, UnknownHostException.class);
if (ignore != null) {
- log.error("Failed to establish connection to server, due to errors with DNS or missing proxy settings.");
+ log.error("Failed to establish connection to server, due to errors with DNS or missing proxy settings.", e);
log.error("Documentation for proxy configuration can be found here: http://apacheignite.readme.io/docs/web-agent#section-proxy-configuration");
System.exit(1);
@@ -323,6 +323,8 @@ public class AgentLauncher {
// Workaround for use self-signed certificate
if (Boolean.getBoolean("trust.all")) {
+ log.info("Trust to all certificates mode is enabled.");
+
SSLContext ctx = SSLContext.getInstance("TLS");
// Create an SSLContext that uses our TrustManager
[37/50] [abbrv] ignite git commit: IGNITE-9151 Web Console: Fixed
logic of removing cluster items: caches, models, igfss.
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/services/Clusters.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/services/Clusters.js b/modules/web-console/frontend/app/services/Clusters.js
index d76f862..c33c145 100644
--- a/modules/web-console/frontend/app/services/Clusters.js
+++ b/modules/web-console/frontend/app/services/Clusters.js
@@ -57,7 +57,9 @@ export default class Clusters {
messageQueueLimit = this.messageQueueLimit.default,
ackSendThreshold = this.ackSendThreshold.default
) => {
- if (currentValue === this.unacknowledgedMessagesBufferSize.default) return currentValue;
+ if (currentValue === this.unacknowledgedMessagesBufferSize.default)
+ return currentValue;
+
const {validRatio} = this.unacknowledgedMessagesBufferSize;
return Math.max(messageQueueLimit * validRatio, ackSendThreshold * validRatio);
},
@@ -253,7 +255,9 @@ export default class Clusters {
maxSize: {
default: '0.2 * totalMemoryAvailable',
min: (dataRegion) => {
- if (!dataRegion) return;
+ if (!dataRegion)
+ return;
+
return dataRegion.initialSize || this.dataRegion.initialSize.default;
}
},
@@ -267,11 +271,14 @@ export default class Clusters {
default: 100,
min: 11,
max: (cluster, dataRegion) => {
- if (!cluster || !dataRegion || !dataRegion.maxSize) return;
+ if (!cluster || !dataRegion || !dataRegion.maxSize)
+ return;
+
const perThreadLimit = 10; // Took from Ignite
const maxSize = dataRegion.maxSize;
const pageSize = cluster.dataStorageConfiguration.pageSize || this.dataStorageConfiguration.pageSize.default;
const maxPoolSize = Math.floor(maxSize / pageSize / perThreadLimit);
+
return maxPoolSize;
}
},
@@ -293,7 +300,10 @@ export default class Clusters {
addDataRegionConfiguration(cluster) {
const dataRegionConfigurations = get(cluster, 'dataStorageConfiguration.dataRegionConfigurations');
- if (!dataRegionConfigurations) return;
+
+ if (!dataRegionConfigurations)
+ return;
+
return dataRegionConfigurations.push(Object.assign(this.makeBlankDataRegionConfiguration(), {
name: uniqueName('New data region', dataRegionConfigurations.concat(cluster.dataStorageConfiguration.defaultDataRegionConfiguration))
}));
@@ -318,7 +328,10 @@ export default class Clusters {
defaultMemoryPolicyExists: (name, items = []) => {
const def = this.memoryPolicy.name.default;
const normalizedName = (name || def);
- if (normalizedName === def) return true;
+
+ if (normalizedName === def)
+ return true;
+
return items.some((policy) => (policy.name || def) === normalizedName);
},
uniqueMemoryPolicyName: (a, items = []) => {
@@ -330,7 +343,9 @@ export default class Clusters {
default: 100,
min: 11,
max: (cluster, memoryPolicy) => {
- if (!memoryPolicy || !memoryPolicy.maxSize) return;
+ if (!memoryPolicy || !memoryPolicy.maxSize)
+ return;
+
const perThreadLimit = 10; // Took from Ignite
const maxSize = memoryPolicy.maxSize;
const pageSize = cluster.memoryConfiguration.pageSize || this.memoryConfiguration.pageSize.default;
@@ -406,7 +421,10 @@ export default class Clusters {
addMemoryPolicy(cluster) {
const memoryPolicies = get(cluster, 'memoryConfiguration.memoryPolicies');
- if (!memoryPolicies) return;
+
+ if (!memoryPolicies)
+ return;
+
return memoryPolicies.push(Object.assign(this.makeBlankMemoryPolicy(), {
// Blank name for default policy if there are not other policies
name: memoryPolicies.length ? uniqueName('New memory policy', memoryPolicies) : ''
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/services/FormUtils.service.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/services/FormUtils.service.js b/modules/web-console/frontend/app/services/FormUtils.service.js
index da6b417..2c81c57 100644
--- a/modules/web-console/frontend/app/services/FormUtils.service.js
+++ b/modules/web-console/frontend/app/services/FormUtils.service.js
@@ -329,12 +329,19 @@ export default ['IgniteFormUtils', ['$window', 'IgniteFocus', '$rootScope', ($wi
function triggerValidation(form) {
const fe = (m) => Object.keys(m.$error)[0];
const em = (e) => (m) => {
- if (!e) return;
+ if (!e)
+ return;
+
const walk = (m) => {
- if (!m.$error[e]) return;
- if (m.$error[e] === true) return m;
+ if (!m.$error[e])
+ return;
+
+ if (m.$error[e] === true)
+ return m;
+
return walk(m.$error[e][0]);
};
+
return walk(m);
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/services/IGFSs.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/services/IGFSs.js b/modules/web-console/frontend/app/services/IGFSs.js
index 87dfd17..de9e43d 100644
--- a/modules/web-console/frontend/app/services/IGFSs.js
+++ b/modules/web-console/frontend/app/services/IGFSs.js
@@ -64,11 +64,15 @@ export default class IGFSs {
secondaryFileSystemEnabled = {
requiredWhenIGFSProxyMode: (igfs) => {
- if (get(igfs, 'defaultMode') === 'PROXY') return get(igfs, 'secondaryFileSystemEnabled') === true;
+ if (get(igfs, 'defaultMode') === 'PROXY')
+ return get(igfs, 'secondaryFileSystemEnabled') === true;
+
return true;
},
requiredWhenPathModeProxyMode: (igfs) => {
- if (get(igfs, 'pathModes', []).some((pm) => pm.mode === 'PROXY')) return get(igfs, 'secondaryFileSystemEnabled') === true;
+ if (get(igfs, 'pathModes', []).some((pm) => pm.mode === 'PROXY'))
+ return get(igfs, 'secondaryFileSystemEnabled') === true;
+
return true;
}
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/services/Models.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/services/Models.js b/modules/web-console/frontend/app/services/Models.js
index 3b714c4..4b360be 100644
--- a/modules/web-console/frontend/app/services/Models.js
+++ b/modules/web-console/frontend/app/services/Models.js
@@ -85,14 +85,19 @@ export default class Models {
* @param {ig.config.model.DomainModel} model
*/
addIndex(model) {
- if (!model) return;
- if (!model.indexes) model.indexes = [];
+ if (!model)
+ return;
+
+ if (!model.indexes)
+ model.indexes = [];
+
model.indexes.push({
_id: ObjectID.generate(),
name: '',
indexType: 'SORTED',
fields: []
});
+
return model.indexes[model.indexes.length - 1];
}
@@ -142,7 +147,9 @@ export default class Models {
*/
indexFieldsHaveUniqueNames: ($value = []) => {
return $value.every((index) => {
- if (!index.fields) return true;
+ if (!index.fields)
+ return true;
+
const uniqueNames = new Set(index.fields.map((ec) => ec.name));
return uniqueNames.size === index.fields.length;
});
@@ -156,7 +163,9 @@ export default class Models {
* @returns {ig.config.model.DomainModel}
*/
removeInvalidFields(model) {
- if (!model) return model;
+ if (!model)
+ return model;
+
const fieldNames = new Set((model.fields || []).map((f) => f.name));
return {
...model,
http://git-wip-us.apache.org/repos/asf/ignite/blob/1cdf7ef9/modules/web-console/frontend/app/services/exceptionHandler.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/services/exceptionHandler.js b/modules/web-console/frontend/app/services/exceptionHandler.js
index 0d9cf3d..82a88e1 100644
--- a/modules/web-console/frontend/app/services/exceptionHandler.js
+++ b/modules/web-console/frontend/app/services/exceptionHandler.js
@@ -19,7 +19,9 @@ import {CancellationError} from 'app/errors/CancellationError';
export function $exceptionHandler($log) {
return function(exception, cause) {
- if (exception instanceof CancellationError) return;
+ if (exception instanceof CancellationError)
+ return;
+
$log.error(exception, cause);
};
}
[33/50] [abbrv] ignite git commit: IGNITE-8179 Resolver do not kill
all server nodes,
ZookeeperDiscoverySpiTest#testCommunicationFailureResolve_KillRandom always
fails on TC - Fixes #4177.
Posted by av...@apache.org.
IGNITE-8179 Resolver do not kill all server nodes, ZookeeperDiscoverySpiTest#testCommunicationFailureResolve_KillRandom always fails on TC - Fixes #4177.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f6f731f5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f6f731f5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f6f731f5
Branch: refs/heads/ignite-8446
Commit: f6f731f575290b10d6d6bcb6869bb0a1b470455e
Parents: 251c91b
Author: Vitaliy Biryukov <Bi...@gmail.com>
Authored: Wed Aug 1 18:26:27 2018 +0300
Committer: Dmitriy Pavlov <dp...@apache.org>
Committed: Wed Aug 1 18:26:27 2018 +0300
----------------------------------------------------------------------
.../zk/internal/ZookeeperDiscoverySpiTest.java | 112 +++++++++++++++----
1 file changed, 90 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f6f731f5/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index 59577c8..24b61d7 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -30,11 +30,11 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
@@ -99,6 +99,7 @@ import org.apache.ignite.internal.util.lang.IgniteInClosure2X;
import org.apache.ignite.internal.util.lang.gridfunc.PredicateMapView;
import org.apache.ignite.internal.util.nio.GridCommunicationClient;
import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.T3;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -179,7 +180,7 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
private static ThreadLocal<Boolean> clientThreadLoc = new ThreadLocal<>();
/** */
- private static ConcurrentHashMap<UUID, Map<Long, DiscoveryEvent>> evts = new ConcurrentHashMap<>();
+ private static ConcurrentHashMap<UUID, Map<T2<Integer, Long>, DiscoveryEvent>> evts = new ConcurrentHashMap<>();
/** */
private static volatile boolean err;
@@ -232,6 +233,9 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
/** */
private String zkRootPath;
+ /** The number of clusters started in one test (increments when the first node in the cluster starts). */
+ private final AtomicInteger clusterNum = new AtomicInteger(0);
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(final String igniteInstanceName) throws Exception {
if (testSockNio)
@@ -304,6 +308,25 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
Map<IgnitePredicate<? extends Event>, int[]> lsnrs = new HashMap<>();
+ if (cfg.isClientMode()) {
+ UUID currNodeId = cfg.getNodeId();
+
+ lsnrs.put(new IgnitePredicate<Event>() {
+ /** Last remembered uuid before node reconnected. */
+ private UUID nodeId = currNodeId;
+
+ @Override public boolean apply(Event evt) {
+ if(evt.type() == EVT_CLIENT_NODE_RECONNECTED){
+ evts.remove(nodeId);
+
+ nodeId = evt.node().id();
+ }
+
+ return false;
+ }
+ }, new int[] {EVT_CLIENT_NODE_RECONNECTED});
+ }
+
lsnrs.put(new IgnitePredicate<Event>() {
/** */
@IgniteInstanceResource
@@ -316,10 +339,10 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
UUID locId = ((IgniteKernal)ignite).context().localNodeId();
- Map<Long, DiscoveryEvent> nodeEvts = evts.get(locId);
+ Map<T2<Integer, Long>, DiscoveryEvent> nodeEvts = evts.get(locId);
if (nodeEvts == null) {
- Object old = evts.put(locId, nodeEvts = new TreeMap<>());
+ Object old = evts.put(locId, nodeEvts = new LinkedHashMap<>());
assertNull(old);
@@ -329,13 +352,18 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
synchronized (nodeEvts) {
DiscoveryLocalJoinData locJoin = ((IgniteEx)ignite).context().discovery().localJoin();
- nodeEvts.put(locJoin.event().topologyVersion(), locJoin.event());
+ if (locJoin.event().node().order() == 1)
+ clusterNum.incrementAndGet();
+
+ nodeEvts.put(new T2<>(clusterNum.get(), locJoin.event().topologyVersion()),
+ locJoin.event());
}
}
}
synchronized (nodeEvts) {
- DiscoveryEvent old = nodeEvts.put(discoveryEvt.topologyVersion(), discoveryEvt);
+ DiscoveryEvent old = nodeEvts.put(new T2<>(clusterNum.get(), discoveryEvt.topologyVersion()),
+ discoveryEvt);
assertNull(old);
}
@@ -2972,8 +3000,6 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
public void testCommunicationFailureResolve_KillRandom() throws Exception {
- fail("https://issues.apache.org/jira/browse/IGNITE-8179");
-
sesTimeout = 2000;
testCommSpi = true;
@@ -2985,6 +3011,10 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
startGridsMultiThreaded(10, 5);
+ int nodesCnt = 15;
+
+ waitForTopology(nodesCnt);
+
int nodeIdx = 15;
for (int i = 0; i < 10; i++) {
@@ -3007,11 +3037,15 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
// No-op.
}
- clientMode(ThreadLocalRandom.current().nextBoolean());
+ boolean clientMode = ThreadLocalRandom.current().nextBoolean();
+
+ clientMode(clientMode);
startGrid(nodeIdx++);
- awaitPartitionMapExchange();
+ nodesCnt = nodesCnt - KillRandomCommunicationFailureResolver.LAST_KILLED_NODES.size() + 1;
+
+ waitForTopology(nodesCnt);
}
}
@@ -4573,13 +4607,13 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
*/
@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
private void checkEventsConsistency() {
- for (Map.Entry<UUID, Map<Long, DiscoveryEvent>> nodeEvtEntry : evts.entrySet()) {
+ for (Map.Entry<UUID, Map<T2<Integer, Long>, DiscoveryEvent>> nodeEvtEntry : evts.entrySet()) {
UUID nodeId = nodeEvtEntry.getKey();
- Map<Long, DiscoveryEvent> nodeEvts = nodeEvtEntry.getValue();
+ Map<T2<Integer, Long>, DiscoveryEvent> nodeEvts = nodeEvtEntry.getValue();
- for (Map.Entry<UUID, Map<Long, DiscoveryEvent>> nodeEvtEntry0 : evts.entrySet()) {
+ for (Map.Entry<UUID, Map<T2<Integer, Long>, DiscoveryEvent>> nodeEvtEntry0 : evts.entrySet()) {
if (!nodeId.equals(nodeEvtEntry0.getKey())) {
- Map<Long, DiscoveryEvent> nodeEvts0 = nodeEvtEntry0.getValue();
+ Map<T2<Integer, Long>, DiscoveryEvent> nodeEvts0 = nodeEvtEntry0.getValue();
synchronized (nodeEvts) {
synchronized (nodeEvts0) {
@@ -4595,20 +4629,39 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
* @param evts1 Received events.
* @param evts2 Received events.
*/
- private void checkEventsConsistency(Map<Long, DiscoveryEvent> evts1, Map<Long, DiscoveryEvent> evts2) {
- for (Map.Entry<Long, DiscoveryEvent> e1 : evts1.entrySet()) {
+ private void checkEventsConsistency(Map<T2<Integer, Long>, DiscoveryEvent> evts1, Map<T2<Integer, Long>, DiscoveryEvent> evts2) {
+ for (Map.Entry<T2<Integer, Long>, DiscoveryEvent> e1 : evts1.entrySet()) {
DiscoveryEvent evt1 = e1.getValue();
DiscoveryEvent evt2 = evts2.get(e1.getKey());
if (evt2 != null) {
assertEquals(evt1.topologyVersion(), evt2.topologyVersion());
- assertEquals(evt1.eventNode(), evt2.eventNode());
- assertEquals(evt1.topologyNodes(), evt2.topologyNodes());
+ assertEquals(evt1.eventNode().consistentId(), evt2.eventNode().consistentId());
+ assertTrue(equalsTopologies(evt1.topologyNodes(), evt2.topologyNodes()));
}
}
}
/**
+ * @param nodes1 Nodes.
+ * @param nodes2 Nodes to be compared with {@code nodes1} for equality.
+ *
+ * @return True if nodes equal by consistent id.
+ */
+ private boolean equalsTopologies(Collection<ClusterNode> nodes1, Collection<ClusterNode> nodes2) {
+ if(nodes1.size() != nodes2.size())
+ return false;
+
+ Set<Object> consistentIds1 = nodes1.stream()
+ .map(ClusterNode::consistentId)
+ .collect(Collectors.toSet());
+
+ return nodes2.stream()
+ .map(ClusterNode::consistentId)
+ .allMatch(consistentIds1::contains);
+ }
+
+ /**
* @param node Node.
* @return Node's discovery SPI.
*/
@@ -4680,7 +4733,7 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
@Override public boolean apply() {
- Map<Long, DiscoveryEvent> nodeEvts = evts.get(nodeId);
+ Map<T2<Integer, Long>, DiscoveryEvent> nodeEvts = evts.get(nodeId);
if (nodeEvts == null) {
info("No events for node: " + nodeId);
@@ -4690,7 +4743,7 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
synchronized (nodeEvts) {
for (DiscoveryEvent expEvt : expEvts) {
- DiscoveryEvent evt0 = nodeEvts.get(expEvt.topologyVersion());
+ DiscoveryEvent evt0 = nodeEvts.get(new T2<>(clusterNum.get(), expEvt.topologyVersion()));
if (evt0 == null) {
info("No event for version: " + expEvt.topologyVersion());
@@ -5197,12 +5250,17 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
}
};
+ /** Last killed nodes. */
+ static final Set<ClusterNode> LAST_KILLED_NODES = new HashSet<>();
+
/** */
@LoggerResource
private IgniteLogger log;
/** {@inheritDoc} */
@Override public void resolve(CommunicationFailureContext ctx) {
+ LAST_KILLED_NODES.clear();
+
List<ClusterNode> nodes = ctx.topologySnapshot();
ThreadLocalRandom rnd = ThreadLocalRandom.current();
@@ -5211,16 +5269,26 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
log.info("Resolver kills nodes [total=" + nodes.size() + ", kill=" + killNodes + ']');
+ long srvCnt = nodes.stream().filter(node -> !node.isClient()).count();
+
Set<Integer> idxs = new HashSet<>();
- while (idxs.size() < killNodes)
- idxs.add(rnd.nextInt(nodes.size()));
+ while (idxs.size() < killNodes) {
+ int idx = rnd.nextInt(nodes.size());
+
+ if(!nodes.get(idx).isClient() && !idxs.contains(idx) && --srvCnt < 1)
+ continue;
+
+ idxs.add(idx);
+ }
for (int idx : idxs) {
ClusterNode node = nodes.get(idx);
log.info("Resolver kills node: " + node.id());
+ LAST_KILLED_NODES.add(node);
+
ctx.killNode(node);
}
}
[10/50] [abbrv] ignite git commit: IGNITE-9114: SQL: fail query after
some timeout if it cannot be mapped to topology. This closes #4453.
Posted by av...@apache.org.
IGNITE-9114: SQL: fail query after some timeout if it cannot be mapped to topology. This closes #4453.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ce30b1e1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ce30b1e1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ce30b1e1
Branch: refs/heads/ignite-8446
Commit: ce30b1e1d921d9c3e1e065f7c01f1f066d42cba4
Parents: d73211d
Author: devozerov <pp...@gmail.com>
Authored: Tue Jul 31 12:53:51 2018 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Tue Jul 31 12:53:51 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 3 +
.../query/h2/opt/GridH2IndexBase.java | 31 +++--
.../query/h2/twostep/GridMapQueryExecutor.java | 75 ++++++++--
.../h2/twostep/GridReduceQueryExecutor.java | 136 +++++++++++++++----
...butedPartitionQueryNodeRestartsSelfTest.java | 21 ++-
...QueryNodeRestartDistributedJoinSelfTest.java | 10 ++
6 files changed, 230 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ce30b1e1/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index eb60371..c4b83e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -480,6 +480,9 @@ public final class IgniteSystemProperties {
/** Disable SQL system views. */
public static final String IGNITE_SQL_DISABLE_SYSTEM_VIEWS = "IGNITE_SQL_DISABLE_SYSTEM_VIEWS";
+ /** SQL retry timeout. */
+ public static final String IGNITE_SQL_RETRY_TIMEOUT = "IGNITE_SQL_RETRY_TIMEOUT";
+
/** Maximum size for affinity assignment history. */
public static final String IGNITE_AFFINITY_HISTORY_SIZE = "IGNITE_AFFINITY_HISTORY_SIZE";
http://git-wip-us.apache.org/repos/asf/ignite/blob/ce30b1e1/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 1c89213..77bd69a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -361,7 +361,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
locNodeHnd,
GridIoPolicy.IDX_POOL,
false))
- throw new GridH2RetryException("Failed to send message to nodes: " + nodes + ".");
+ throw retryException("Failed to send message to nodes: " + nodes);
}
/**
@@ -554,14 +554,15 @@ public abstract class GridH2IndexBase extends BaseIndex {
ClusterNode node = ctx.discovery().node(nodeId);
if (node == null)
- throw new GridH2RetryException("Failed to find node.");
+ throw retryException("Failed to get node by ID during broadcast [nodeId=" + nodeId + ']');
nodes.add(node);
}
}
if (F.isEmpty(nodes))
- throw new GridH2RetryException("Failed to collect affinity nodes.");
+ throw retryException("Failed to collect affinity nodes during broadcast [" +
+ "cacheName=" + cctx.name() + ']');
}
int segmentsCount = segmentsCount();
@@ -615,7 +616,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
node = cctx.affinity().primaryByKey(affKeyObj, qctx.topologyVersion());
if (node == null) // Node was not found, probably topology changed and we need to retry the whole query.
- throw new GridH2RetryException("Failed to find node.");
+ throw retryException("Failed to get primary node by key for range segment.");
}
return new SegmentKey(node, segmentForPartition(partition));
@@ -1317,10 +1318,10 @@ public abstract class GridH2IndexBase extends BaseIndex {
for (int attempt = 0;; attempt++) {
if (qctx.isCleared())
- throw new GridH2RetryException("Query is cancelled.");
+ throw retryException("Query is cancelled.");
if (kernalContext().isStopping())
- throw new GridH2RetryException("Stopping node.");
+ throw retryException("Local node is stopping.");
GridH2IndexRangeResponse res;
@@ -1328,7 +1329,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
res = respQueue.poll(500, TimeUnit.MILLISECONDS);
}
catch (InterruptedException ignored) {
- throw new GridH2RetryException("Interrupted.");
+ throw retryException("Interrupted while waiting for reply.");
}
if (res != null) {
@@ -1355,10 +1356,10 @@ public abstract class GridH2IndexBase extends BaseIndex {
case STATUS_NOT_FOUND:
if (req == null || req.bounds() == null) // We have already received the first response.
- throw new GridH2RetryException("Failure on remote node.");
+ throw retryException("Failure on remote node.");
if (U.currentTimeMillis() - start > 30_000)
- throw new GridH2RetryException("Timeout.");
+ throw retryException("Timeout reached.");
try {
U.sleep(20 * attempt);
@@ -1381,7 +1382,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
}
if (!kernalContext().discovery().alive(node))
- throw new GridH2RetryException("Node left: " + node);
+ throw retryException("Node has left topology: " + node.id());
}
}
@@ -1585,6 +1586,16 @@ public abstract class GridH2IndexBase extends BaseIndex {
}
/**
+ * Create retry exception for distributed join.
+ *
+ * @param msg Message.
+ * @return Exception.
+ */
+ private GridH2RetryException retryException(String msg) {
+ return new GridH2RetryException(msg);
+ }
+
+ /**
*
*/
private static final class CursorIteratorWrapper implements Iterator<GridH2Row> {
http://git-wip-us.apache.org/repos/asf/ignite/blob/ce30b1e1/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index 930ada2..216a259 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -53,6 +53,7 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionsReservation;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
import org.apache.ignite.internal.processors.cache.query.CacheQueryType;
@@ -293,6 +294,8 @@ public class GridMapQueryExecutor {
* @param topVer Topology version.
* @param explicitParts Explicit partitions list.
* @param reserved Reserved list.
+ * @param nodeId Node ID.
+ * @param reqId Request ID.
* @return {@code true} If all the needed partitions successfully reserved.
* @throws IgniteCheckedException If failed.
*/
@@ -300,7 +303,9 @@ public class GridMapQueryExecutor {
@Nullable List<Integer> cacheIds,
AffinityTopologyVersion topVer,
final int[] explicitParts,
- List<GridReservable> reserved
+ List<GridReservable> reserved,
+ UUID nodeId,
+ long reqId
) throws IgniteCheckedException {
assert topVer != null;
@@ -312,8 +317,14 @@ public class GridMapQueryExecutor {
for (int i = 0; i < cacheIds.size(); i++) {
GridCacheContext<?, ?> cctx = ctx.cache().context().cacheContext(cacheIds.get(i));
- if (cctx == null) // Cache was not found, probably was not deployed yet.
+ // Cache was not found, probably was not deployed yet.
+ if (cctx == null) {
+ logRetry("Failed to reserve partitions for query (cache is not found on local node) [" +
+ "rmtNodeId=" + nodeId + ", reqId=" + reqId + ", affTopVer=" + topVer + ", cacheId=" +
+ cacheIds.get(i) + "]");
+
return false;
+ }
if (cctx.isLocal() || !cctx.rebalanceEnabled())
continue;
@@ -325,8 +336,13 @@ public class GridMapQueryExecutor {
if (explicitParts == null && r != null) { // Try to reserve group partition if any and no explicits.
if (r != MapReplicatedReservation.INSTANCE) {
- if (!r.reserve())
+ if (!r.reserve()) {
+ logRetry("Failed to reserve partitions for query (group reservation failed) [" +
+ "rmtNodeId=" + nodeId + ", reqId=" + reqId + ", affTopVer=" + topVer +
+ ", cacheId=" + cacheIds.get(i) + ", cacheName=" + cctx.name() + "]");
+
return false; // We need explicit partitions here -> retry.
+ }
reserved.add(r);
}
@@ -340,8 +356,17 @@ public class GridMapQueryExecutor {
GridDhtLocalPartition part = partition(cctx, p);
// We don't need to reserve partitions because they will not be evicted in replicated caches.
- if (part == null || part.state() != OWNING)
+ GridDhtPartitionState partState = part != null ? part.state() : null;
+
+ if (partState != OWNING) {
+ logRetry("Failed to reserve partitions for query (partition of " +
+ "REPLICATED cache is not in OWNING state) [rmtNodeId=" + nodeId +
+ ", reqId=" + reqId + ", affTopVer=" + topVer + ", cacheId=" + cacheIds.get(i) +
+ ", cacheName=" + cctx.name() + ", part=" + p + ", partFound=" + (part != null) +
+ ", partState=" + partState + "]");
+
return false;
+ }
}
// Mark that we checked this replicated cache.
@@ -355,14 +380,31 @@ public class GridMapQueryExecutor {
for (int partId : partIds) {
GridDhtLocalPartition part = partition(cctx, partId);
- if (part == null || part.state() != OWNING || !part.reserve())
+ GridDhtPartitionState partState = part != null ? part.state() : null;
+
+ if (partState != OWNING || !part.reserve()) {
+ logRetry("Failed to reserve partitions for query (partition of " +
+ "PARTITIONED cache cannot be reserved) [rmtNodeId=" + nodeId + ", reqId=" + reqId +
+ ", affTopVer=" + topVer + ", cacheId=" + cacheIds.get(i) +
+ ", cacheName=" + cctx.name() + ", part=" + partId + ", partFound=" + (part != null) +
+ ", partState=" + partState + "]");
+
return false;
+ }
reserved.add(part);
// Double check that we are still in owning state and partition contents are not cleared.
- if (part.state() != OWNING)
+ partState = part.state();
+
+ if (part.state() != OWNING) {
+ logRetry("Failed to reserve partitions for query (partition of " +
+ "PARTITIONED cache is not in OWNING state after reservation) [rmtNodeId=" + nodeId +
+ ", reqId=" + reqId + ", affTopVer=" + topVer + ", cacheId=" + cacheIds.get(i) +
+ ", cacheName=" + cctx.name() + ", part=" + partId + ", partState=" + partState + "]");
+
return false;
+ }
}
if (explicitParts == null) {
@@ -388,6 +430,15 @@ public class GridMapQueryExecutor {
}
/**
+ * Load failed partition reservation.
+ *
+ * @param msg Message.
+ */
+ private void logRetry(String msg) {
+ log.info(msg);
+ }
+
+ /**
* @param ints Integers.
* @return Collection wrapper.
*/
@@ -622,7 +673,7 @@ public class GridMapQueryExecutor {
try {
if (topVer != null) {
// Reserve primary for topology version or explicit partitions.
- if (!reservePartitions(cacheIds, topVer, parts, reserved)) {
+ if (!reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId)) {
// Unregister lazy worker because re-try may never reach this node again.
if (lazy)
stopAndUnregisterCurrentLazyWorker();
@@ -739,8 +790,14 @@ public class GridMapQueryExecutor {
if (lazy)
stopAndUnregisterCurrentLazyWorker();
- if (X.hasCause(e, GridH2RetryException.class))
+ GridH2RetryException retryErr = X.cause(e, GridH2RetryException.class);
+
+ if (retryErr != null) {
+ logRetry("Failed to execute non-collocated query (will retry) [nodeId=" + node.id() +
+ ", reqId=" + reqId + ", errMsg=" + retryErr.getMessage() + ']');
+
sendRetry(node, reqId, segmentId);
+ }
else {
U.error(log, "Failed to execute local query.", e);
@@ -788,7 +845,7 @@ public class GridMapQueryExecutor {
List<GridReservable> reserved = new ArrayList<>();
- if (!reservePartitions(cacheIds, topVer, parts, reserved)) {
+ if (!reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId)) {
U.error(log, "Failed to reserve partitions for DML request. [localNodeId=" + ctx.localNodeId() +
", nodeId=" + node.id() + ", reqId=" + req.requestId() + ", cacheIds=" + cacheIds +
", topVer=" + topVer + ", parts=" + Arrays.toString(parts) + ']');
http://git-wip-us.apache.org/repos/asf/ignite/blob/ce30b1e1/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index cd76bc1..d778fcc 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -45,6 +45,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteClientDisconnectedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cache.query.QueryCancelledException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.events.DiscoveryEvent;
@@ -101,6 +102,7 @@ import org.h2.value.Value;
import org.jetbrains.annotations.Nullable;
import static java.util.Collections.singletonList;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT;
import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
import static org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery.EMPTY_PARAMS;
import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.OFF;
@@ -111,6 +113,9 @@ import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySpl
* Reduce query executor.
*/
public class GridReduceQueryExecutor {
+ /** Fail query after 10 seconds of unsuccessful attempts to reserve partitions. */
+ public static final long DFLT_RETRY_TIMEOUT = 10_000L;
+
/** */
private static final String MERGE_INDEX_UNSORTED = "merge_scan";
@@ -442,14 +447,24 @@ public class GridReduceQueryExecutor {
}
/**
+ * Load failed partition reservation.
+ *
+ * @param msg Message.
+ */
+ private void logRetry(String msg) {
+ log.info(msg);
+ }
+
+ /**
* @param isReplicatedOnly If we must only have replicated caches.
* @param topVer Topology version.
* @param cacheIds Participating cache IDs.
* @param parts Partitions.
+ * @param qryId Query ID.
* @return Data nodes or {@code null} if repartitioning started and we need to retry.
*/
private Map<ClusterNode, IntArray> stableDataNodes(boolean isReplicatedOnly, AffinityTopologyVersion topVer,
- List<Integer> cacheIds, int[] parts) {
+ List<Integer> cacheIds, int[] parts, long qryId) {
GridCacheContext<?, ?> cctx = cacheContext(cacheIds.get(0));
// If the first cache is not partitioned, find it (if it's present) and move it to index 0.
@@ -507,8 +522,15 @@ public class GridReduceQueryExecutor {
disjoint = !extraNodes.equals(nodes);
if (disjoint) {
- if (isPreloadingActive(cacheIds))
+ if (isPreloadingActive(cacheIds)) {
+ logRetry("Failed to calculate nodes for SQL query (got disjoint node map during rebalance) " +
+ "[qryId=" + qryId + ", affTopVer=" + topVer + ", cacheIds=" + cacheIds +
+ ", parts=" + (parts == null ? "[]" : Arrays.toString(parts)) +
+ ", replicatedOnly=" + isReplicatedOnly + ", lastCache=" + extraCctx.name() +
+ ", lastCacheId=" + extraCctx.cacheId() + ']');
+
return null; // Retry.
+ }
else
throw new CacheException("Caches have distinct sets of data nodes [cache1=" + cctx.name() +
", cache2=" + extraCacheName + "]");
@@ -546,8 +568,14 @@ public class GridReduceQueryExecutor {
final boolean isReplicatedOnly = qry.isReplicatedOnly();
- // Fail if all caches are replicated and explicit partitions are set.
+ long retryTimeout = retryTimeout();
+
+ final long startTime = U.currentTimeMillis();
+
for (int attempt = 0;; attempt++) {
+ if (attempt > 0 && retryTimeout > 0 && (U.currentTimeMillis() - startTime > retryTimeout))
+ throw new CacheException("Failed to map SQL query to topology.");
+
if (attempt != 0) {
try {
Thread.sleep(attempt * 10); // Wait for exchange.
@@ -559,7 +587,7 @@ public class GridReduceQueryExecutor {
}
}
- final long qryReqId = qryIdGen.incrementAndGet();
+ long qryReqId = qryIdGen.incrementAndGet();
final ReduceQueryRun r = new ReduceQueryRun(qryReqId, qry.originalSql(), schemaName,
h2.connectionForSchema(schemaName), qry.mapQueries().size(), qry.pageSize(),
@@ -602,7 +630,8 @@ public class GridReduceQueryExecutor {
if (qry.isLocal())
nodes = singletonList(ctx.discovery().localNode());
else {
- NodesForPartitionsResult nodesParts = nodesForPartitions(cacheIds, topVer, parts, isReplicatedOnly);
+ NodesForPartitionsResult nodesParts =
+ nodesForPartitions(cacheIds, topVer, parts, isReplicatedOnly, qryReqId);
nodes = nodesParts.nodes();
partsMap = nodesParts.partitionsMap();
@@ -704,9 +733,11 @@ public class GridReduceQueryExecutor {
final boolean distributedJoins = qry.distributedJoins();
+ final long qryReqId0 = qryReqId;
+
cancel.set(new Runnable() {
@Override public void run() {
- send(finalNodes, new GridQueryCancelRequest(qryReqId), null, false);
+ send(finalNodes, new GridQueryCancelRequest(qryReqId0), null, false);
}
});
@@ -885,10 +916,10 @@ public class GridReduceQueryExecutor {
) {
AffinityTopologyVersion topVer = h2.readyTopologyVersion();
- NodesForPartitionsResult nodesParts = nodesForPartitions(cacheIds, topVer, parts, isReplicatedOnly);
-
final long reqId = qryIdGen.incrementAndGet();
+ NodesForPartitionsResult nodesParts = nodesForPartitions(cacheIds, topVer, parts, isReplicatedOnly, reqId);
+
final GridRunningQueryInfo qryInfo = new GridRunningQueryInfo(reqId, selectQry, GridCacheQueryType.SQL_FIELDS,
schemaName, U.currentTimeMillis(), cancel, false);
@@ -1128,9 +1159,10 @@ public class GridReduceQueryExecutor {
* Calculates data nodes for replicated caches on unstable topology.
*
* @param cacheIds Cache IDs.
+ * @param qryId Query ID.
* @return Collection of all data nodes owning all the caches or {@code null} for retry.
*/
- private Collection<ClusterNode> replicatedUnstableDataNodes(List<Integer> cacheIds) {
+ private Collection<ClusterNode> replicatedUnstableDataNodes(List<Integer> cacheIds, long qryId) {
int i = 0;
GridCacheContext<?, ?> cctx = cacheContext(cacheIds.get(i++));
@@ -1145,7 +1177,7 @@ public class GridReduceQueryExecutor {
assert cctx.isReplicated(): "all the extra caches must be replicated here";
}
- Set<ClusterNode> nodes = replicatedUnstableDataNodes(cctx);
+ Set<ClusterNode> nodes = replicatedUnstableDataNodes(cctx, qryId);
if (F.isEmpty(nodes))
return null; // Retry.
@@ -1161,15 +1193,20 @@ public class GridReduceQueryExecutor {
"with tables in partitioned caches [replicatedCache=" + cctx.name() + ", " +
"partitionedCache=" + extraCctx.name() + "]");
- Set<ClusterNode> extraOwners = replicatedUnstableDataNodes(extraCctx);
+ Set<ClusterNode> extraOwners = replicatedUnstableDataNodes(extraCctx, qryId);
if (F.isEmpty(extraOwners))
return null; // Retry.
nodes.retainAll(extraOwners);
- if (nodes.isEmpty())
+ if (nodes.isEmpty()) {
+ logRetry("Failed to calculate nodes for SQL query (got disjoint node map for REPLICATED caches " +
+ "during rebalance) [qryId=" + qryId + ", cacheIds=" + cacheIds +
+ ", lastCache=" + extraCctx.name() + ", lastCacheId=" + extraCctx.cacheId() + ']');
+
return null; // Retry.
+ }
}
return nodes;
@@ -1190,9 +1227,10 @@ public class GridReduceQueryExecutor {
* Collects all the nodes owning all the partitions for the given replicated cache.
*
* @param cctx Cache context.
+ * @param qryId Query ID.
* @return Owning nodes or {@code null} if we can't find owners for some partitions.
*/
- private Set<ClusterNode> replicatedUnstableDataNodes(GridCacheContext<?,?> cctx) {
+ private Set<ClusterNode> replicatedUnstableDataNodes(GridCacheContext<?,?> cctx, long qryId) {
assert cctx.isReplicated() : cctx.name() + " must be replicated";
String cacheName = cctx.name();
@@ -1206,13 +1244,23 @@ public class GridReduceQueryExecutor {
for (int p = 0, parts = cctx.affinity().partitions(); p < parts; p++) {
List<ClusterNode> owners = cctx.topology().owners(p);
- if (F.isEmpty(owners))
+ if (F.isEmpty(owners)) {
+ logRetry("Failed to calculate nodes for SQL query (partition of a REPLICATED cache has no owners) [" +
+ "qryId=" + qryId + ", cacheName=" + cctx.name() + ", cacheId=" + cctx.cacheId() +
+ ", part=" + p + ']');
+
return null; // Retry.
+ }
dataNodes.retainAll(owners);
- if (dataNodes.isEmpty())
+ if (dataNodes.isEmpty()) {
+ logRetry("Failed to calculate nodes for SQL query (partitions of a REPLICATED has no common owners) [" +
+ "qryId=" + qryId + ", cacheName=" + cctx.name() + ", cacheId=" + cctx.cacheId() +
+ ", lastPart=" + p + ']');
+
return null; // Retry.
+ }
}
return dataNodes;
@@ -1222,10 +1270,11 @@ public class GridReduceQueryExecutor {
* Calculates partition mapping for partitioned cache on unstable topology.
*
* @param cacheIds Cache IDs.
+ * @param qryId Query ID.
* @return Partition mapping or {@code null} if we can't calculate it due to repartitioning and we need to retry.
*/
@SuppressWarnings("unchecked")
- private Map<ClusterNode, IntArray> partitionedUnstableDataNodes(List<Integer> cacheIds) {
+ private Map<ClusterNode, IntArray> partitionedUnstableDataNodes(List<Integer> cacheIds, long qryId) {
// If the main cache is replicated, just replace it with the first partitioned.
GridCacheContext<?,?> cctx = findFirstPartitioned(cacheIds);
@@ -1260,8 +1309,14 @@ public class GridReduceQueryExecutor {
continue;
}
- else if (!F.isEmpty(dataNodes(cctx.groupId(), NONE)))
+ else if (!F.isEmpty(dataNodes(cctx.groupId(), NONE))) {
+ logRetry("Failed to calculate nodes for SQL query (partition has no owners, but corresponding " +
+ "cache group has data nodes) [qryId=" + qryId + ", cacheIds=" + cacheIds +
+ ", cacheName=" + cctx.name() + ", cacheId=" + cctx.cacheId() + ", part=" + p +
+ ", cacheGroupId=" + cctx.groupId() + ']');
+
return null; // Retry.
+ }
throw new CacheException("Failed to find data nodes [cache=" + cctx.name() + ", part=" + p + "]");
}
@@ -1289,8 +1344,15 @@ public class GridReduceQueryExecutor {
continue; // Skip unmapped partitions.
if (F.isEmpty(owners)) {
- if (!F.isEmpty(dataNodes(extraCctx.groupId(), NONE)))
+ if (!F.isEmpty(dataNodes(extraCctx.groupId(), NONE))) {
+ logRetry("Failed to calculate nodes for SQL query (partition has no owners, but " +
+ "corresponding cache group has data nodes) [qryId=" + qryId +
+ ", cacheIds=" + cacheIds + ", cacheName=" + extraCctx.name() +
+ ", cacheId=" + extraCctx.cacheId() + ", part=" + p +
+ ", cacheGroupId=" + extraCctx.groupId() + ']');
+
return null; // Retry.
+ }
throw new CacheException("Failed to find data nodes [cache=" + extraCctx.name() +
", part=" + p + "]");
@@ -1301,8 +1363,14 @@ public class GridReduceQueryExecutor {
else {
partLocs[p].retainAll(owners); // Intersection of owners.
- if (partLocs[p].isEmpty())
+ if (partLocs[p].isEmpty()) {
+ logRetry("Failed to calculate nodes for SQL query (caches have no common data nodes for " +
+ "partition) [qryId=" + qryId + ", cacheIds=" + cacheIds +
+ ", lastCacheName=" + extraCctx.name() + ", lastCacheId=" + extraCctx.cacheId() +
+ ", part=" + p + ']');
+
return null; // Intersection is empty -> retry.
+ }
}
}
}
@@ -1314,19 +1382,29 @@ public class GridReduceQueryExecutor {
if (!extraCctx.isReplicated())
continue;
- Set<ClusterNode> dataNodes = replicatedUnstableDataNodes(extraCctx);
+ Set<ClusterNode> dataNodes = replicatedUnstableDataNodes(extraCctx, qryId);
if (F.isEmpty(dataNodes))
return null; // Retry.
+ int part = 0;
+
for (Set<ClusterNode> partLoc : partLocs) {
if (partLoc == UNMAPPED_PARTS)
continue; // Skip unmapped partition.
partLoc.retainAll(dataNodes);
- if (partLoc.isEmpty())
+ if (partLoc.isEmpty()) {
+ logRetry("Failed to calculate nodes for SQL query (caches have no common data nodes for " +
+ "partition) [qryId=" + qryId + ", cacheIds=" + cacheIds +
+ ", lastReplicatedCacheName=" + extraCctx.name() +
+ ", lastReplicatedCacheId=" + extraCctx.cacheId() + ", part=" + part + ']');
+
return null; // Retry.
+ }
+
+ part++;
}
}
}
@@ -1475,19 +1553,20 @@ public class GridReduceQueryExecutor {
* @param topVer Topology version.
* @param parts Partitions array.
* @param isReplicatedOnly Allow only replicated caches.
+ * @param qryId Query ID.
* @return Result.
*/
private NodesForPartitionsResult nodesForPartitions(List<Integer> cacheIds, AffinityTopologyVersion topVer,
- int[] parts, boolean isReplicatedOnly) {
+ int[] parts, boolean isReplicatedOnly, long qryId) {
Collection<ClusterNode> nodes = null;
Map<ClusterNode, IntArray> partsMap = null;
Map<ClusterNode, IntArray> qryMap = null;
if (isPreloadingActive(cacheIds)) {
if (isReplicatedOnly)
- nodes = replicatedUnstableDataNodes(cacheIds);
+ nodes = replicatedUnstableDataNodes(cacheIds, qryId);
else {
- partsMap = partitionedUnstableDataNodes(cacheIds);
+ partsMap = partitionedUnstableDataNodes(cacheIds, qryId);
if (partsMap != null) {
qryMap = narrowForQuery(partsMap, parts);
@@ -1497,7 +1576,7 @@ public class GridReduceQueryExecutor {
}
}
else {
- qryMap = stableDataNodes(isReplicatedOnly, topVer, cacheIds, parts);
+ qryMap = stableDataNodes(isReplicatedOnly, topVer, cacheIds, parts, qryId);
if (qryMap != null)
nodes = qryMap.keySet();
@@ -1676,6 +1755,13 @@ public class GridReduceQueryExecutor {
return cp.isEmpty() ? null : cp;
}
+ /**
+ * @return Query retry timeout.
+ */
+ private static long retryTimeout() {
+ return IgniteSystemProperties.getLong(IGNITE_SQL_RETRY_TIMEOUT, DFLT_RETRY_TIMEOUT);
+ }
+
/** */
private static class ExplicitPartitionsSpecializer implements IgniteBiClosure<ClusterNode, Message, Message> {
/** Partitions map. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/ce30b1e1/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java
index 8525410..806fbbf 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java
@@ -24,14 +24,31 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerArray;
import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor;
/**
* Tests distributed queries over set of partitions on unstable topology.
*/
-public class IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest
- extends IgniteCacheDistributedPartitionQueryAbstractSelfTest {
+public class IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest extends
+ IgniteCacheDistributedPartitionQueryAbstractSelfTest {
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ System.setProperty(IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT, Long.toString(1000_000L));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ System.setProperty(IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT,
+ Long.toString(GridReduceQueryExecutor.DFLT_RETRY_TIMEOUT));
+
+ super.afterTestsStopped();
+ }
+
/**
* Tests join query within region on unstable topology.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/ce30b1e1/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartDistributedJoinSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartDistributedJoinSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartDistributedJoinSelfTest.java
index 4f20078..bad5303 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartDistributedJoinSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartDistributedJoinSelfTest.java
@@ -19,8 +19,10 @@ package org.apache.ignite.internal.processors.cache.distributed.near;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor;
import org.apache.ignite.internal.util.GridRandom;
import org.apache.ignite.internal.util.typedef.CAX;
import org.apache.ignite.internal.util.typedef.X;
@@ -41,6 +43,8 @@ public class IgniteCacheQueryNodeRestartDistributedJoinSelfTest extends IgniteCa
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
+ System.setProperty(IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT, Long.toString(1000_000L));
+
super.beforeTestsStarted();
if (totalNodes > GRID_CNT) {
@@ -51,6 +55,12 @@ public class IgniteCacheQueryNodeRestartDistributedJoinSelfTest extends IgniteCa
totalNodes = GRID_CNT;
}
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ System.setProperty(IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT,
+ Long.toString(GridReduceQueryExecutor.DFLT_RETRY_TIMEOUT));
+ }
+
/**
* @throws Exception If failed.
*/
[50/50] [abbrv] ignite git commit: Merge branch 'ignite-8446-master'
into ignite-8446
Posted by av...@apache.org.
Merge branch 'ignite-8446-master' into ignite-8446
Signed-off-by: Anton Vinogradov <av...@apache.org>
# Conflicts:
# modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/fa1a33ec
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/fa1a33ec
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/fa1a33ec
Branch: refs/heads/ignite-8446
Commit: fa1a33ec6dac5f276629ece9b3f039f78e2b3595
Parents: c2426df ff7372b
Author: Anton Vinogradov <av...@apache.org>
Authored: Fri Aug 3 12:57:59 2018 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Fri Aug 3 12:57:59 2018 +0300
----------------------------------------------------------------------
README.md | 2 +-
assembly/LICENSE_FABRIC | 2 +-
docker/web-agent/Dockerfile | 3 +-
examples/pom.xml | 21 -
.../ml/tutorial/Step_8_CV_with_Param_Grid.java | 173 ++
.../cluster/TensorFlowClusterExample.java | 118 --
.../test/config/aop/aspectj/META-INF/aop.xml | 4 +-
modules/aws/pom.xml | 18 -
.../apache/ignite/IgniteSystemProperties.java | 6 +
.../org/apache/ignite/cache/CacheMetrics.java | 77 +
.../internal/commandline/CommandHandler.java | 162 +-
.../commandline/cache/CacheArguments.java | 34 +
.../delta/DataPageInsertFragmentRecord.java | 2 +-
.../cache/CacheClusterMetricsMXBeanImpl.java | 55 +
.../cache/CacheLocalMetricsMXBeanImpl.java | 55 +
.../processors/cache/CacheMetricsImpl.java | 206 ++
.../processors/cache/CacheMetricsSnapshot.java | 157 ++
.../processors/cache/GridCacheAdapter.java | 67 +-
.../processors/cache/GridCacheContext.java | 4 +
.../processors/cache/GridCacheEntryEx.java | 8 +-
.../cache/GridCacheLoaderWriterStore.java | 6 +
.../processors/cache/GridCacheMapEntry.java | 126 +-
.../GridCachePartitionExchangeManager.java | 68 +-
.../processors/cache/GridCachePreloader.java | 21 +-
.../cache/GridCachePreloaderAdapter.java | 6 +
.../cache/GridCacheUpdateAtomicResult.java | 15 +-
.../cache/IgniteCacheOffheapManagerImpl.java | 3 +
.../CacheDataStructuresManager.java | 5 +-
.../dht/GridDhtPartitionTopologyImpl.java | 23 +-
.../distributed/dht/GridDhtTxPrepareFuture.java | 5 +
.../GridDhtAtomicAbstractUpdateFuture.java | 8 +-
.../GridDhtAtomicAbstractUpdateRequest.java | 16 +-
.../dht/atomic/GridDhtAtomicCache.java | 63 +-
.../GridDhtAtomicSingleUpdateRequest.java | 11 +-
.../dht/atomic/GridDhtAtomicUpdateRequest.java | 5 +-
.../dht/preloader/GridDhtPartitionDemander.java | 55 +-
.../dht/preloader/GridDhtPartitionSupplier.java | 26 +-
.../dht/preloader/GridDhtPreloader.java | 60 +-
.../preloader/GridDhtPreloaderAssignments.java | 6 +-
.../preloader/latch/ExchangeLatchManager.java | 12 +-
.../distributed/near/GridNearAtomicCache.java | 14 +-
.../distributed/near/GridNearGetFuture.java | 2 +-
.../local/atomic/GridLocalAtomicCache.java | 116 +-
.../GridCacheDatabaseSharedManager.java | 46 +-
.../persistence/freelist/AbstractFreeList.java | 2 +-
.../persistence/freelist/CacheFreeListImpl.java | 11 +
.../persistence/tree/io/AbstractDataPageIO.java | 23 +-
.../continuous/CacheContinuousQueryManager.java | 20 +-
.../cache/transactions/IgniteTxAdapter.java | 2 +
.../cache/transactions/IgniteTxHandler.java | 20 +-
.../cache/transactions/IgniteTxManager.java | 11 +-
.../cache/verify/IdleVerifyDumpResult.java | 73 +
.../cache/verify/IdleVerifyResultV2.java | 85 +-
.../verify/VerifyBackupPartitionsDumpTask.java | 230 ++
.../cluster/DiscoveryDataClusterState.java | 7 -
.../cluster/GridClusterStateProcessor.java | 2 +-
.../continuous/GridContinuousProcessor.java | 72 +-
.../platform/cache/PlatformCache.java | 11 +
.../internal/processors/query/QueryUtils.java | 3 +
.../service/GridServiceProcessor.java | 5 +-
.../visor/baseline/VisorBaselineTask.java | 2 +-
.../visor/baseline/VisorBaselineViewTask.java | 73 +
.../internal/visor/tx/VisorTxTaskArg.java | 1 -
.../internal/visor/tx/VisorTxTaskResult.java | 7 -
.../visor/verify/VisorIdleVerifyDumpTask.java | 37 +
.../verify/VisorIdleVerifyDumpTaskArg.java | 73 +
.../visor/verify/VisorIdleVerifyJob.java | 83 +
.../resources/META-INF/classnames.properties | 2 +
.../ClusterBaselineNodesMetricsSelfTest.java | 1 -
.../cache/CacheGetEntryAbstractTest.java | 2 +-
.../cache/CacheValidatorMetricsTest.java | 4 +-
.../cache/GridCacheAbstractMetricsSelfTest.java | 417 ++++
.../processors/cache/GridCacheTestEntryEx.java | 8 +-
.../IgniteCacheEntryListenerAbstractTest.java | 16 +-
.../IgniteClusterActivateDeactivateTest.java | 50 +
...sExchangeOnDiscoveryHistoryOverflowTest.java | 2 -
.../CacheLateAffinityAssignmentTest.java | 2 -
.../dht/GridCacheDhtPreloadSelfTest.java | 68 +-
.../atomic/IgniteCacheAtomicProtocolTest.java | 3 -
.../GridCacheNearAtomicMetricsSelfTest.java | 71 +
.../near/GridCacheNearMetricsSelfTest.java | 388 ++++
.../GridCacheRebalancingAsyncSelfTest.java | 7 +-
.../GridCacheRebalancingCancelTest.java | 106 +
...idCacheRebalancingPartitionCountersTest.java | 3 +-
.../GridCacheRebalancingSyncSelfTest.java | 149 +-
...ReplicatedFullApiMultithreadedSelfTest1.java | 1980 ------------------
.../PageEvictionMultinodeMixedRegionsTest.java | 2 +
.../IgnitePdsCorruptedStoreTest.java | 99 +-
...lientAffinityAssignmentWithBaselineTest.java | 8 +-
.../IgniteBaselineAbstractFullApiSelfTest.java | 2 +
...SlowHistoricalRebalanceSmallHistoryTest.java | 5 +-
...nitePdsCacheDestroyDuringCheckpointTest.java | 4 +-
.../db/file/IgnitePdsPageReplacementTest.java | 2 +
...lFlushMultiNodeFailoverAbstractSelfTest.java | 2 +-
...usQueryFactoryFilterRandomOperationTest.java | 6 +-
...acheContinuousQueryRandomOperationsTest.java | 15 +-
.../CacheContinuousQueryVariationsTest.java | 12 +-
...inuousWithTransformerReplicatedSelfTest.java | 2 +-
...ridCacheContinuousQueryAbstractSelfTest.java | 6 +-
.../GridMarshallerMappingConsistencyTest.java | 3 +-
.../GridP2PContinuousDeploymentSelfTest.java | 70 +-
.../platform/PlatformCacheWriteMetricsTask.java | 55 +
.../testframework/junits/GridAbstractTest.java | 95 +-
.../junits/common/GridCommonAbstractTest.java | 117 +-
.../ignite/testsuites/IgniteBasicTestSuite.java | 12 +-
.../IgniteBasicWithPersistenceTestSuite.java | 67 +
...IgniteBinaryObjectsComputeGridTestSuite.java | 3 +
.../IgniteCacheEvictionSelfTestSuite.java | 3 -
.../IgniteCacheFullApiSelfTestSuite.java | 3 +
.../IgniteCacheMetricsSelfTestSuite.java | 6 +-
.../ignite/testsuites/IgniteCacheTestSuite.java | 8 +-
.../testsuites/IgniteCacheTestSuite2.java | 5 +-
.../testsuites/IgniteCacheTestSuite3.java | 9 +-
.../testsuites/IgniteCacheTestSuite4.java | 5 +-
.../testsuites/IgniteCacheTestSuite5.java | 6 +-
.../testsuites/IgniteCacheTestSuite6.java | 27 +-
.../testsuites/IgniteCacheTestSuite7.java | 53 +-
.../testsuites/IgniteComputeGridTestSuite.java | 2 -
.../testsuites/IgniteKernalSelfTestSuite.java | 7 +-
.../IgniteMarshallerSelfTestSuite.java | 5 +-
.../testsuites/IgniteUtilSelfTestSuite.java | 7 +-
.../ignite/util/GridCommandHandlerTest.java | 178 +-
.../GridInternalTaskUnusedWalSegmentsTest.java | 27 +-
.../apache/ignite/sink/flink/IgniteSink.java | 88 +-
.../sink/flink/FlinkIgniteSinkSelfTest.java | 154 +-
.../flink/src/test/resources/example-ignite.xml | 7 +-
modules/gce/pom.xml | 2 +-
.../processors/query/h2/IgniteH2Indexing.java | 85 +-
.../query/h2/ddl/DdlStatementsProcessor.java | 28 +-
.../query/h2/dml/UpdatePlanBuilder.java | 15 +-
.../query/h2/opt/GridH2IndexBase.java | 31 +-
.../query/h2/sql/GridSqlQuerySplitter.java | 12 +-
.../processors/query/h2/sys/SqlSystemIndex.java | 143 ++
.../processors/query/h2/sys/SqlSystemTable.java | 208 ++
.../query/h2/sys/SqlSystemTableEngine.java | 60 +
.../h2/sys/view/SqlAbstractLocalSystemView.java | 104 +
.../h2/sys/view/SqlAbstractSystemView.java | 134 ++
.../query/h2/sys/view/SqlSystemView.java | 79 +
.../sys/view/SqlSystemViewColumnCondition.java | 102 +
.../query/h2/sys/view/SqlSystemViewNodes.java | 116 +
.../query/h2/twostep/GridMapQueryExecutor.java | 75 +-
.../h2/twostep/GridReduceQueryExecutor.java | 140 +-
.../processors/cache/BigEntryQueryTest.java | 150 ++
...butedPartitionQueryNodeRestartsSelfTest.java | 21 +-
...QueryNodeRestartDistributedJoinSelfTest.java | 10 +
...ngBaselineCacheQueryNodeRestartSelfTest.java | 2 +
...eBaselineCacheQueryNodeRestartsSelfTest.java | 2 +
.../query/SqlSystemViewsSelfTest.java | 247 +++
.../IgniteBinaryCacheQueryTestSuite.java | 2 +
.../IgniteCacheQuerySelfTestSuite.java | 2 +
...acheWithIndexingAndPersistenceTestSuite.java | 38 +
.../IgniteCacheWithIndexingTestSuite.java | 6 +-
modules/kafka/pom.xml | 12 +-
.../ignite/stream/kafka/KafkaStreamer.java | 173 +-
.../kafka/KafkaIgniteStreamerSelfTest.java | 39 +-
.../ignite/stream/kafka/TestKafkaBroker.java | 18 +-
modules/kubernetes/pom.xml | 8 +-
.../TcpDiscoveryKubernetesIpFinder.java | 4 +-
.../ignite/ml/selection/cv/CrossValidation.java | 212 +-
.../ml/selection/cv/CrossValidationResult.java | 134 ++
.../ml/selection/paramgrid/ParamGrid.java | 58 +
.../paramgrid/ParameterSetGenerator.java | 91 +
.../ml/selection/paramgrid/package-info.java | 22 +
.../org/apache/ignite/ml/tree/DecisionTree.java | 6 +-
.../tree/DecisionTreeClassificationTrainer.java | 38 +
.../ignite/ml/selection/SelectionTestSuite.java | 2 +
.../paramgrid/ParameterSetGeneratorTest.java | 56 +
.../osgi-karaf/src/main/resources/features.xml | 10 +-
.../cpp/core-test/src/continuous_query_test.cpp | 6 +-
.../Cache/CacheMetricsTest.cs | 11 +
.../Continuous/ContinuousQueryAbstractTest.cs | 10 +-
.../Apache.Ignite.Core/Cache/ICacheMetrics.cs | 88 +
.../Impl/Cache/CacheMetricsImpl.cs | 79 +-
.../Impl/Cache/Event/CacheEntryRemoveEvent.cs | 4 +-
.../Query/Continuous/ContinuousQueryUtils.cs | 2 +-
modules/rest-http/pom.xml | 6 +-
modules/spark/pom.xml | 6 -
modules/tensorflow/README.txt | 55 +-
modules/tensorflow/pom.xml | 56 +-
modules/tensorflow/src/main/assembly/zip.xml | 56 +
.../cluster/TensorFlowClusterGateway.java | 34 +-
.../TensorFlowClusterGatewayManager.java | 95 +-
.../cluster/TensorFlowClusterMaintainer.java | 105 +-
.../cluster/TensorFlowClusterManager.java | 268 ++-
.../cluster/TensorFlowJobArchive.java | 65 +
.../cluster/spec/TensorFlowClusterSpec.java | 39 +
.../spec/TensorFlowServerAddressSpec.java | 14 +
.../tfrunning/TensorFlowServerManager.java | 112 +-
.../TensorFlowServerScriptFormatter.java | 62 +
.../cluster/util/ClusterPortManager.java | 168 +-
.../cluster/util/TensorFlowChiefRunner.java | 80 +
.../cluster/util/TensorFlowClusterResolver.java | 92 +-
.../util/TensorFlowUserScriptRunner.java | 236 +++
.../ignite/tensorflow/core/ProcessManager.java | 5 +-
.../tensorflow/core/ProcessManagerWrapper.java | 5 +-
.../longrunning/LongRunningProcessManager.java | 19 +-
.../task/LongRunningProcessStartTask.java | 4 +-
.../core/nativerunning/NativeProcess.java | 6 +-
.../nativerunning/NativeProcessManager.java | 12 +-
.../task/NativeProcessStartTask.java | 81 +-
.../PythonProcessBuilderSupplier.java | 57 +
.../pythonrunning/PythonProcessManager.java | 35 +-
.../core/util/AsyncNativeProcessRunner.java | 107 +
.../core/util/NativeProcessRunner.java | 133 ++
.../tensorflow/submitter/JobSubmitter.java | 35 +
.../submitter/command/AbstractCommand.java | 55 +
.../submitter/command/AttachCommand.java | 51 +
.../tensorflow/submitter/command/PsCommand.java | 47 +
.../submitter/command/RootCommand.java | 42 +
.../submitter/command/StartCommand.java | 205 ++
.../submitter/command/StopCommand.java | 50 +
.../submitter/command/package-info.java | 22 +
.../tensorflow/submitter/package-info.java | 23 +
.../tensorflow/util/SerializableConsumer.java | 29 +
.../tensorflow/util/SerializableSupplier.java | 29 +
.../ignite/tensorflow/util/package-info.java | 22 +
modules/tensorflow/src/main/sh/ignite-tf.sh | 19 +
modules/tensorflow/src/main/sh/logback.xml | 36 +
.../LongRunningProcessManagerTest.java | 10 +-
modules/web-console/e2e/docker-compose.yml | 2 +-
.../testcafe/components/pageConfiguration.js | 2 +-
.../configuration/clusterFormChangeDetection.js | 12 +-
.../PageConfigurationAdvancedCluster.js | 11 +
modules/web-console/frontend/app/app.js | 10 +
.../frontend/app/browserUpdate/index.js | 8 +-
.../app/components/bs-select-menu/index.js | 6 +-
.../components/bs-select-menu/strip.filter.js | 22 +
.../app/components/bs-select-menu/template.pug | 2 +-
.../transcludeToBody.directive.js | 2 +-
.../components/cluster-selector/controller.js | 13 +-
.../components/cluster-selector/template.pug | 8 +-
.../form-field/showValidationError.directive.js | 6 +-
.../grid-column-selector/controller.js | 8 +-
.../app/components/grid-export/index.js | 2 +-
.../app/components/grid-export/style.scss | 34 +
.../app/components/grid-export/template.pug | 6 +-
.../components/grid-item-selected/template.pug | 5 +-
.../app/components/grid-no-data/controller.js | 7 +-
.../components/grid-showing-rows/component.js | 29 +
.../components/grid-showing-rows/controller.js | 47 +
.../app/components/grid-showing-rows/index.js | 24 +
.../app/components/grid-showing-rows/style.scss | 20 +
.../ignite-chart-series-selector/component.js | 28 +
.../ignite-chart-series-selector/controller.js | 62 +
.../ignite-chart-series-selector/index.js | 24 +
.../ignite-chart-series-selector/template.pug | 29 +
.../app/components/ignite-chart/controller.js | 320 +++
.../app/components/ignite-chart/index.js | 38 +
.../app/components/ignite-chart/style.scss | 69 +
.../app/components/ignite-chart/template.pug | 36 +
.../list-editable-save-on-changes/directives.js | 14 +-
.../list-editable-transclude/directive.js | 12 +-
.../app/components/list-editable/controller.js | 6 +-
.../components/cluster-edit-form/controller.js | 2 -
.../templates/load-balancing.pug | 2 +-
.../cluster-edit-form/templates/memory.pug | 2 +-
.../cluster-edit-form/templates/swap.pug | 2 +-
.../components/model-edit-form/controller.js | 10 +-
.../page-configure-basic/controller.js | 13 +-
.../pco-grid-column-categories/directive.js | 4 +-
.../components/formUICanExitGuard.js | 12 +-
.../components/modal-import-models/component.js | 23 +-
.../tables-action-cell/component.js | 14 +-
.../modal-preview-project/controller.js | 10 +-
.../components/pc-form-field-size/controller.js | 8 +-
.../components/pc-items-table/controller.js | 26 +-
.../components/pc-ui-grid-filters/directive.js | 4 +-
.../components/pc-ui-grid-filters/index.js | 5 +-
.../components/pcIsInCollection.js | 4 +-
.../page-configure/components/pcValidation.js | 8 +-
.../app/components/page-configure/index.js | 14 +-
.../app/components/page-configure/reducer.js | 90 +-
.../page-configure/services/ConfigureState.js | 5 +
.../app/components/page-configure/states.js | 88 +-
.../components/page-configure/store/effects.js | 114 +-
.../page-configure/store/selectors.js | 50 +-
.../transitionHooks/errorState.js | 5 +-
.../components/page-password-changed/style.scss | 2 +-
.../components/queries-notebook/controller.js | 33 +-
.../queries-notebook/template.tpl.pug | 4 +-
.../queries-notebooks-list/controller.js | 4 +-
.../app/components/page-queries/index.js | 2 +
.../app/components/page-queries/style.scss | 20 +
.../frontend/app/components/page-signin/run.js | 5 +-
.../components/panel-collapsible/controller.js | 19 +-
.../app/components/ui-grid-filters/directive.js | 3 +-
.../app/components/ui-grid-filters/index.js | 4 +-
.../frontend/app/filters/bytes.filter.js | 34 +
.../frontend/app/filters/bytes.filter.spec.js | 36 +
.../generator/ConfigurationGenerator.js | 3 +-
.../field/bs-select-placeholder.directive.js | 4 +-
.../modules/form/validator/unique.directive.js | 6 +-
.../modules/nodes/nodes-dialog.controller.js | 6 +-
.../app/modules/nodes/nodes-dialog.scss | 17 +-
.../app/modules/nodes/nodes-dialog.tpl.pug | 28 +-
.../frontend/app/primitives/btn/index.scss | 9 +
.../app/primitives/form-field/checkbox.pug | 2 +-
.../app/primitives/form-field/index.scss | 46 +-
.../app/primitives/form-field/number.pug | 5 +-
.../web-console/frontend/app/services/Caches.js | 10 +-
.../frontend/app/services/Clusters.js | 38 +-
.../app/services/ErrorParser.service.js | 89 +
.../frontend/app/services/FormUtils.service.js | 13 +-
.../web-console/frontend/app/services/IGFSs.js | 8 +-
.../frontend/app/services/Messages.service.js | 29 +-
.../web-console/frontend/app/services/Models.js | 17 +-
.../frontend/app/services/exceptionHandler.js | 4 +-
modules/web-console/frontend/package.json | 4 +-
.../frontend/public/stylesheets/style.scss | 23 +-
.../frontend/public/stylesheets/variables.scss | 6 +-
.../frontend/views/sql/cache-metadata.tpl.pug | 3 +-
.../frontend/views/sql/chart-settings.tpl.pug | 2 +-
modules/web-console/web-agent/pom.xml | 2 +-
.../ignite/console/agent/AgentLauncher.java | 6 +-
.../console/agent/handlers/ClusterListener.java | 7 +
modules/zookeeper/pom.xml | 10 +-
.../zk/TcpDiscoveryZookeeperIpFinder.java | 2 +-
.../zk/ZookeeperDiscoverySpiTestSuite1.java | 1 +
.../zk/ZookeeperDiscoverySpiTestSuite2.java | 3 +
.../zk/internal/ZookeeperDiscoverySpiTest.java | 128 +-
parent/pom.xml | 5 +-
321 files changed, 10710 insertions(+), 4227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/fa1a33ec/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/fa1a33ec/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/fa1a33ec/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index 1f498d0,7a5aa80..0b259f9
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@@ -54,9 -44,7 +44,10 @@@ import org.apache.ignite.internal.proce
import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNoDeadlockDetectionTest;
import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest;
import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTopologyChangeTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnIncorrectParamsTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxStateChangeEventTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxWithSmallTimeoutAndContentionOneKeyTest;
+ import org.apache.ignite.testframework.junits.GridAbstractTest;
/**
* Test suite.