You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/07/05 11:24:15 UTC
[33/33] ignite git commit: Merge.
Merge.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5b2400ab
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5b2400ab
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5b2400ab
Branch: refs/heads/ignite-2.1.2-exchange
Commit: 5b2400ab4a0c43a1b8597fd674d5c6a373bd3c43
Parents: 377cc9d aa21a9b
Author: sboikov <sb...@gridgain.com>
Authored: Wed Jul 5 14:22:31 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Wed Jul 5 14:22:31 2017 +0300
----------------------------------------------------------------------
.../jdbc2/JdbcPreparedStatementSelfTest.java | 35 +
.../jdbc/JdbcPreparedStatementSelfTest.java | 35 +
.../thin/JdbcThinPreparedStatementSelfTest.java | 35 +
.../PersistentStoreConfiguration.java | 39 +-
.../org/apache/ignite/events/EventType.java | 12 +
.../ignite/events/WalSegmentArchivedEvent.java | 62 +
.../apache/ignite/internal/GridComponent.java | 4 +-
.../ignite/internal/GridPluginComponent.java | 2 +-
.../apache/ignite/internal/IgniteKernal.java | 33 +-
.../internal/jdbc/JdbcPreparedStatement.java | 6 +-
.../internal/jdbc/thin/JdbcThinConnection.java | 7 +-
.../jdbc/thin/JdbcThinPreparedStatement.java | 2 -
.../internal/jdbc2/JdbcPreparedStatement.java | 12 +-
.../internal/managers/GridManagerAdapter.java | 2 +-
.../checkpoint/GridCheckpointManager.java | 2 +-
.../managers/communication/GridIoManager.java | 10 +-
.../communication/GridMessageListener.java | 3 +-
.../deployment/GridDeploymentCommunication.java | 4 +-
.../internal/managers/discovery/DiscoCache.java | 17 +-
.../discovery/DiscoveryLocalJoinData.java | 104 ++
.../discovery/GridDiscoveryManager.java | 128 +-
.../eventstorage/GridEventStorageManager.java | 4 +-
.../pagemem/store/IgnitePageStoreManager.java | 3 +-
.../internal/pagemem/wal/record/WALRecord.java | 11 +-
.../processors/GridProcessorAdapter.java | 2 +-
.../cache/CacheAffinitySharedManager.java | 67 +-
.../processors/cache/CacheGroupContext.java | 4 +-
.../processors/cache/CacheGroupData.java | 4 +-
.../cache/ChangeGlobalStateMessage.java | 120 --
.../processors/cache/ClusterCachesInfo.java | 493 +++++--
.../internal/processors/cache/ClusterState.java | 38 -
.../cache/DynamicCacheChangeRequest.java | 52 +-
.../processors/cache/ExchangeActions.java | 37 +-
.../processors/cache/GridCacheEventManager.java | 2 -
.../cache/GridCacheEvictionManager.java | 1 -
.../processors/cache/GridCacheIoManager.java | 98 +-
.../processors/cache/GridCacheMvccManager.java | 9 +-
.../GridCachePartitionExchangeManager.java | 439 +++---
.../processors/cache/GridCacheProcessor.java | 193 +--
.../cache/GridCacheSharedContext.java | 60 +-
.../cache/GridCacheSharedManager.java | 6 -
.../cache/GridCacheSharedManagerAdapter.java | 16 -
.../cache/GridDeferredAckMessageSender.java | 3 +-
.../processors/cache/PendingDiscoveryEvent.java | 61 +
.../processors/cache/StateChangeRequest.java | 77 ++
.../cache/binary/BinaryMetadataTransport.java | 4 +-
.../binary/CacheObjectBinaryProcessorImpl.java | 4 +-
.../distributed/GridCacheTxRecoveryFuture.java | 1 -
.../distributed/dht/GridDhtCacheAdapter.java | 1 -
.../cache/distributed/dht/GridDhtGetFuture.java | 1 -
.../distributed/dht/GridDhtGetSingleFuture.java | 2 -
.../dht/GridDhtPartitionTopologyImpl.java | 13 +-
.../dht/GridDhtTopologyFutureAdapter.java | 2 +-
.../dht/GridPartitionedSingleGetFuture.java | 3 -
.../GridNearAtomicAbstractUpdateFuture.java | 1 -
.../dht/preloader/GridDhtForceKeysFuture.java | 1 -
.../dht/preloader/GridDhtPartitionDemander.java | 2 +
.../GridDhtPartitionsExchangeFuture.java | 240 +++-
.../preloader/GridDhtPartitionsFullMessage.java | 44 +-
.../GridDhtPartitionsSingleMessage.java | 38 +-
.../dht/preloader/GridDhtPreloader.java | 2 +-
.../distributed/near/GridNearGetFuture.java | 2 -
.../near/GridNearTxPrepareRequest.java | 1 -
.../GridCacheDatabaseSharedManager.java | 111 +-
.../persistence/GridCacheOffheapManager.java | 5 +-
.../IgniteCacheDatabaseSharedManager.java | 74 +-
.../persistence/IgniteCacheSnapshotManager.java | 20 +-
.../persistence/file/FilePageStoreManager.java | 14 +-
.../persistence/tree/io/TrackingPageIO.java | 12 +-
.../wal/AbstractWalRecordsIterator.java | 289 ++++
.../cache/persistence/wal/FileInput.java | 16 +-
.../cache/persistence/wal/FileWALPointer.java | 4 +-
.../wal/FileWriteAheadLogManager.java | 594 ++++----
.../cache/persistence/wal/RecordSerializer.java | 5 +
.../persistence/wal/SegmentArchiveResult.java | 61 +
.../persistence/wal/SegmentEofException.java | 3 +-
.../wal/reader/IgniteWalIteratorFactory.java | 102 ++
.../wal/reader/StandaloneGridKernalContext.java | 499 +++++++
...ndaloneIgniteCacheDatabaseSharedManager.java | 30 +
.../reader/StandaloneWalRecordsIterator.java | 258 ++++
.../wal/serializer/RecordV1Serializer.java | 45 +-
.../query/GridCacheDistributedQueryManager.java | 4 +-
.../store/GridCacheStoreManagerAdapter.java | 1 -
.../cache/transactions/IgniteTxManager.java | 2 +-
.../cache/version/GridCacheVersionManager.java | 6 -
.../cacheobject/IgniteCacheObjectProcessor.java | 5 -
.../IgniteCacheObjectProcessorImpl.java | 5 -
.../cluster/ChangeGlobalStateFinishMessage.java | 86 ++
.../cluster/ChangeGlobalStateMessage.java | 140 ++
.../processors/cluster/ClusterProcessor.java | 5 +-
.../cluster/DiscoveryDataClusterState.java | 157 +++
.../cluster/GridClusterStateProcessor.java | 1129 ++++++---------
.../cluster/IgniteChangeGlobalStateSupport.java | 3 +-
.../continuous/GridContinuousProcessor.java | 4 +-
.../datastreamer/DataStreamProcessor.java | 2 +-
.../datastreamer/DataStreamerImpl.java | 2 +-
.../datastructures/DataStructuresProcessor.java | 6 +-
.../datastructures/GridCacheAtomicLongImpl.java | 2 +-
.../GridCacheAtomicReferenceImpl.java | 2 +-
.../GridCacheAtomicSequenceImpl.java | 2 +-
.../GridCacheAtomicStampedImpl.java | 2 +-
.../GridCacheCountDownLatchImpl.java | 2 +-
.../datastructures/GridCacheLockImpl.java | 4 +-
.../datastructures/GridCacheQueueAdapter.java | 1 -
.../datastructures/GridCacheSemaphoreImpl.java | 2 +-
.../datastructures/GridCacheSetImpl.java | 1 -
.../processors/igfs/IgfsDataManager.java | 2 +-
.../igfs/IgfsFragmentizerManager.java | 4 +-
.../internal/processors/igfs/IgfsImpl.java | 2 -
.../internal/processors/igfs/IgfsProcessor.java | 2 +-
.../processors/job/GridJobProcessor.java | 8 +-
.../GridMarshallerMappingProcessor.java | 4 +-
.../utils/PlatformConfigurationUtils.java | 8 +-
.../processors/query/GridQueryProcessor.java | 6 +-
.../processors/rest/GridRestProcessor.java | 2 +-
.../cluster/GridChangeStateCommandHandler.java | 2 +-
.../handlers/task/GridTaskCommandHandler.java | 4 +-
.../service/GridServiceProcessor.java | 6 +-
.../processors/task/GridTaskProcessor.java | 8 +-
.../visor/cache/VisorCacheConfiguration.java | 11 +-
.../visor/node/VisorBasicConfiguration.java | 2 +-
.../node/VisorMemoryPolicyConfiguration.java | 2 +-
.../visor/node/VisorNodeDataCollectorJob.java | 32 +-
.../node/VisorNodeDataCollectorJobResult.java | 73 +-
.../visor/node/VisorNodeDataCollectorTask.java | 14 +-
.../node/VisorNodeDataCollectorTaskResult.java | 28 +-
.../visor/node/VisorPersistenceMetrics.java | 214 +++
.../visor/service/VisorServiceTask.java | 10 +-
.../jobstealing/JobStealingCollisionSpi.java | 2 +-
.../ignite/spi/discovery/tcp/ClientImpl.java | 12 +-
.../ignite/spi/discovery/tcp/ServerImpl.java | 10 +-
.../internal/TestRecordingCommunicationSpi.java | 10 +
...GridManagerLocalMessageListenerSelfTest.java | 4 +-
...idCommunicationManagerListenersSelfTest.java | 2 +-
.../GridCommunicationSendMessageSelfTest.java | 2 +-
...unicationBalanceMultipleConnectionsTest.java | 5 +
.../cache/GridCacheAbstractMetricsSelfTest.java | 24 -
.../cache/GridCachePartitionedGetSelfTest.java | 2 +-
.../cache/IgniteActiveClusterTest.java | 182 ---
.../IgniteClusterActivateDeactivateTest.java | 1284 ++++++++++++++++++
...erActivateDeactivateTestWithPersistence.java | 197 +++
.../IgniteDaemonNodeMarshallerCacheTest.java | 10 -
...lerCacheClientRequestsMappingOnMissTest.java | 6 +-
...naryObjectMetadataExchangeMultinodeTest.java | 6 +-
.../IgniteSemaphoreAbstractSelfTest.java | 17 +-
...IgnitePersistentStoreDataStructuresTest.java | 2 +
.../wal/IgniteWalHistoryReservationsTest.java | 2 +-
.../db/wal/reader/IgniteWalReaderTest.java | 385 ++++++
.../db/wal/reader/MockWalIteratorFactory.java | 114 ++
.../pagemem/NoOpPageStoreManager.java | 12 +-
.../persistence/pagemem/NoOpWALManager.java | 23 +-
.../AbstractNodeJoinTemplate.java | 149 +-
.../IgniteChangeGlobalStateAbstractTest.java | 65 +-
.../IgniteChangeGlobalStateCacheTest.java | 2 +-
...IgniteChangeGlobalStateDataStreamerTest.java | 5 +-
...gniteChangeGlobalStateDataStructureTest.java | 6 +-
.../IgniteChangeGlobalStateFailOverTest.java | 26 +-
.../IgniteChangeGlobalStateTest.java | 158 +--
.../IgniteStandByClusterTest.java | 17 +-
.../join/JoinActiveNodeToActiveCluster.java | 62 +-
...ctiveNodeToActiveClusterWithPersistence.java | 17 +
.../IgniteStandByClientReconnectTest.java | 13 +-
...eStandByClientReconnectToNewClusterTest.java | 13 +-
...DeadlockDetectionMessageMarshallingTest.java | 2 +-
.../communication/GridIoManagerBenchmark.java | 4 +-
.../communication/GridIoManagerBenchmark0.java | 12 +-
.../communication/GridCacheMessageSelfTest.java | 2 +-
...cpCommunicationSpiMultithreadedSelfTest.java | 2 +-
.../testframework/GridSpiTestContext.java | 5 +-
.../testframework/junits/GridAbstractTest.java | 4 +-
.../junits/common/GridCommonAbstractTest.java | 3 +
.../ignite/testsuites/IgnitePdsTestSuite2.java | 9 +-
.../testsuites/IgniteStandByClusterSuite.java | 5 +-
.../processors/hadoop/HadoopProcessor.java | 4 +-
.../hadoop/shuffle/HadoopShuffle.java | 2 +-
...ileSystemShmemExternalDualAsyncSelfTest.java | 5 +
.../query/h2/opt/GridH2IndexBase.java | 2 +-
.../query/h2/twostep/GridMapQueryExecutor.java | 2 +-
.../h2/twostep/GridReduceQueryExecutor.java | 2 +-
.../cache/IgniteCacheAbstractQuerySelfTest.java | 3 +-
.../Cache/CacheConfigurationTest.cs | 21 +
.../Apache.Ignite.Core.Tests/EventsTest.cs | 19 +-
.../Cache/Configuration/CacheConfiguration.cs | 75 +-
.../Apache.Ignite.Core/IgniteConfiguration.cs | 7 +-
.../Impl/Events/RemoteListenEventFilter.cs | 3 +
.../Apache.Ignite.Core/Impl/IgniteUtils.cs | 12 +-
.../spark/JavaEmbeddedIgniteRDDSelfTest.java | 5 +
.../frontend/app/filters/duration.filter.js | 2 +-
.../generator/ConfigurationGenerator.js | 9 +-
.../states/configuration/caches/affinity.pug | 18 +-
.../states/configuration/caches/general.pug | 50 +-
191 files changed, 7182 insertions(+), 2830 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b2400ab/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 6301497,8d08c3f..a8af753
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@@ -53,7 -53,8 +54,9 @@@ import org.apache.ignite.internal.proce
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.util.GridAtomicLong;
+ import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
+ import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b2400ab/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b2400ab/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index e238b11,624dec0..dab678a
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@@ -2170,9 -2148,10 +2150,12 @@@ public class GridCacheProcessor extend
if (!sharedCtx.kernalContext().clientNode())
sharedCtx.database().onCacheGroupsStopped(stoppedGroups);
+
+ if (exchActions.deactivate())
+ sharedCtx.deactivate();
}
+
+ log.info("processor onExchangeDone [topVer=" + topVer + ", time=" + (U.currentTimeMillis() - start) + ']');
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b2400ab/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 4e46daa,cea758a..3479b4e
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@@ -77,9 -73,9 +77,10 @@@ import org.apache.ignite.internal.proce
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
- import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
+ import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
+ import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
+import org.apache.ignite.internal.util.GridAtomicLong;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@@ -578,9 -571,12 +592,13 @@@ public class GridDhtPartitionsExchangeF
if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) {
DiscoveryCustomMessage msg = ((DiscoveryCustomEvent)discoEvt).customMessage();
- if (msg instanceof DynamicCacheChangeBatch) {
- long start = U.currentTimeMillis();
+ if (msg instanceof ChangeGlobalStateMessage) {
+ assert exchActions != null && !exchActions.empty();
+ exchange = onClusterStateChangeRequest(crdNode);
+ }
+ else if (msg instanceof DynamicCacheChangeBatch) {
++ long start = U.currentTimeMillis();
assert exchActions != null && !exchActions.empty();
exchange = onCacheChangeRequest(crdNode);
@@@ -609,12 -601,28 +627,30 @@@
cctx.affinity().initStartedCaches(crdNode, this, receivedCaches);
}
- else
- cctx.cache().startCachesOnLocalJoin(topVer);
+ else {
+ cctx.activate();
+
+ List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> caches =
+ cctx.cache().cachesToStartOnLocalJoin();
+
+ if (cctx.database().persistenceEnabled() &&
+ !cctx.kernalContext().clientNode()) {
+ List<DynamicCacheDescriptor> startDescs = new ArrayList<>();
+
+ if (caches != null) {
+ for (T2<DynamicCacheDescriptor, NearCacheConfiguration> c : caches)
+ startDescs.add(c.get1());
+ }
+
+ cctx.database().readCheckpointAndRestoreMemory(startDescs);
+ }
+
+ cctx.cache().startCachesOnLocalJoin(caches, topVer);
+ }
}
+ exchLog.info("Caches start time [topVer=" + topologyVersion() + ", time=" + (U.currentTimeMillis() - start) + ']');
+
exchange = CU.clientNode(discoEvt.eventNode()) ?
onClientNodeEvent(crdNode) :
onServerNodeEvent(crdNode);
@@@ -1227,11 -1285,10 +1333,12 @@@
assert !nodes.contains(cctx.localNode());
+ sndTs = U.currentTimeMillis();
+
- if (log.isDebugEnabled())
+ if (log.isDebugEnabled()) {
log.debug("Sending full partition map [nodeIds=" + F.viewReadOnly(nodes, F.node2id()) +
", exchId=" + exchId + ", msg=" + m + ']');
+ }
for (ClusterNode node : nodes) {
try {
@@@ -1586,18 -1568,9 +1694,18 @@@
if (updateSingleMap) {
try {
- // Do not update partition map, in case cluster transitioning to inactive state.
+ cnt.incrementAndGet();
+
+ long start = U.currentTimeMillis();
+
- if (!exchangeOnChangeGlobalState || exchActions.newClusterState() != ClusterState.INACTIVE)
+ if (!deactivateCluster())
updatePartitionSingleMap(node, msg);
+
+ long time = U.currentTimeMillis() - start;
+
+ minTime.setIfLess(time);
+ maxTime.setIfGreater(time);
+ totTime.addAndGet(time);
}
finally {
synchronized (this) {
@@@ -2068,14 -2048,17 +2195,19 @@@
}
}
+ rcvTs = U.currentTimeMillis();
+
updatePartitionFullMap(msg);
- if (exchangeOnChangeGlobalState && !F.isEmpty(msg.getExceptionsMap()))
- cctx.kernalContext().state().onFullResponseMessage(msg.getExceptionsMap());
+ IgniteCheckedException err = null;
- onDone(exchId.topologyVersion());
+ if (stateChangeExchange() && !F.isEmpty(msg.getErrorsMap())) {
+ err = new IgniteCheckedException("Cluster state change failed");
+
+ cctx.kernalContext().state().onStateChangeError(msg.getErrorsMap(), exchActions.stateChangeRequest());
+ }
+
+ onDone(exchId.topologyVersion(), err);
}
/**