You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2017/07/21 13:24:55 UTC

[3/5] ignite git commit: IGNITE-5797 - WIP

IGNITE-5797 - WIP


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/75037190
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/75037190
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/75037190

Branch: refs/heads/ignite-5797
Commit: 750371903f88341fdc1d1b8e0e2554a4383e6e10
Parents: 70d0f99
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Fri Jul 21 11:49:17 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Jul 21 11:49:17 2017 +0300

----------------------------------------------------------------------
 .../ignite/internal/GridKernalContext.java      |   9 +-
 .../ignite/internal/GridKernalContextImpl.java  |  17 ++-
 .../apache/ignite/internal/IgniteKernal.java    |   2 +
 .../managers/communication/GridIoManager.java   |  21 ++-
 .../communication/GridIoMessageFactory.java     |   6 +
 .../processors/cache/GridCacheIoManager.java    |   3 +-
 .../GridDistributedTxFinishRequest.java         |  25 +++-
 .../GridDistributedTxFinishResponse.java        |  22 ++-
 .../GridDistributedTxPrepareRequest.java        |  25 +++-
 .../GridDistributedTxPrepareResponse.java       |  30 ++++-
 .../distributed/dht/GridDhtTxFinishFuture.java  |  27 +++-
 .../distributed/dht/GridDhtTxFinishRequest.java |  15 ++-
 .../dht/GridDhtTxFinishResponse.java            |  13 +-
 .../cache/distributed/dht/GridDhtTxLocal.java   |  16 ++-
 .../distributed/dht/GridDhtTxLocalAdapter.java  |  32 ++++-
 .../distributed/dht/GridDhtTxPrepareFuture.java |  10 +-
 .../dht/GridDhtTxPrepareRequest.java            |  12 +-
 .../dht/GridDhtTxPrepareResponse.java           |  11 +-
 ...arOptimisticSerializableTxPrepareFuture.java |   6 +-
 .../near/GridNearOptimisticTxPrepareFuture.java |   6 +-
 .../GridNearPessimisticTxPrepareFuture.java     |   6 +-
 .../near/GridNearTxFinishFuture.java            |  30 +++--
 .../near/GridNearTxFinishRequest.java           |  10 +-
 .../near/GridNearTxFinishResponse.java          |  10 +-
 .../cache/distributed/near/GridNearTxLocal.java |   8 +-
 .../near/GridNearTxPrepareRequest.java          |   9 +-
 .../near/GridNearTxPrepareResponse.java         |   8 +-
 .../wal/reader/StandaloneGridKernalContext.java |   6 +
 .../cache/transactions/IgniteTxHandler.java     |  19 ++-
 .../processors/trace/IgniteTraceAware.java      |  42 ++++++
 .../internal/processors/trace/NodeTrace.java    | 133 +++++++++++++++++++
 .../processors/trace/TraceProcessor.java        |  43 ++++++
 .../internal/processors/trace/TraceUtils.java   |  25 ++++
 .../ignite/internal/util/nio/GridNioServer.java |   4 +
 .../trace/GridCacheTxTracingSelfTest.java       |  80 +++++++++++
 35 files changed, 669 insertions(+), 72 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index ef4001f..18a3729 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -33,7 +33,6 @@ import org.apache.ignite.internal.managers.failover.GridFailoverManager;
 import org.apache.ignite.internal.managers.indexing.GridIndexingManager;
 import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager;
 import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
-import org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
 import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.closure.GridClosureProcessor;
@@ -64,6 +63,7 @@ import org.apache.ignite.internal.processors.service.GridServiceProcessor;
 import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
 import org.apache.ignite.internal.processors.task.GridTaskProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
+import org.apache.ignite.internal.processors.trace.TraceProcessor;
 import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
 import org.apache.ignite.internal.util.IgniteExceptionRegistry;
 import org.apache.ignite.internal.util.StripedExecutor;
@@ -420,6 +420,13 @@ public interface GridKernalContext extends Iterable<GridComponent> {
     public DataStructuresProcessor dataStructures();
 
     /**
+     * Gets trace processor.
+     *
+     * @return Trace processor.
+     */
+    public TraceProcessor trace();
+
+    /**
      * Sets segmented flag to {@code true} when node is stopped due to segmentation issues.
      */
     public void markSegmented();

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 42a9b2c..58522b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -47,7 +47,6 @@ import org.apache.ignite.internal.managers.indexing.GridIndexingManager;
 import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager;
 import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
 import org.apache.ignite.internal.processors.cache.CacheConflictResolutionManager;
-import org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
 import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
 import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
@@ -81,6 +80,7 @@ import org.apache.ignite.internal.processors.service.GridServiceProcessor;
 import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
 import org.apache.ignite.internal.processors.task.GridTaskProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
+import org.apache.ignite.internal.processors.trace.TraceProcessor;
 import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
 import org.apache.ignite.internal.util.IgniteExceptionRegistry;
 import org.apache.ignite.internal.util.StripedExecutor;
@@ -285,6 +285,10 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
 
     /** */
     @GridToStringExclude
+    private TraceProcessor traceProc;
+
+    /** */
+    @GridToStringExclude
     private List<GridComponent> comps = new LinkedList<>();
 
     /** */
@@ -580,8 +584,10 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
             poolProc = (PoolProcessor) comp;
         else if (comp instanceof GridMarshallerMappingProcessor)
             mappingProc = (GridMarshallerMappingProcessor)comp;
+        else if (comp instanceof TraceProcessor)
+            traceProc = (TraceProcessor)comp;
         else if (!(comp instanceof DiscoveryNodeValidationProcessor
-                || comp instanceof PlatformPluginProcessor))
+            || comp instanceof PlatformPluginProcessor))
             assert (comp instanceof GridPluginComponent) : "Unknown manager class: " + comp.getClass();
 
         if (addToList)
@@ -835,6 +841,11 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     }
 
     /** {@inheritDoc} */
+    @Override public TraceProcessor trace() {
+        return traceProc;
+    }
+
+    /** {@inheritDoc} */
     @Override public IgniteLogger log(String ctgr) {
         return config().getGridLogger().getLogger(ctgr);
     }
@@ -1006,7 +1017,7 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     }
 
     /** {@inheritDoc} */
-    public Map<String, ? extends ExecutorService> customExecutors() {
+    @Override public Map<String, ? extends ExecutorService> customExecutors() {
         return customExecSvcs;
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 00c1d73..15ddb77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -147,6 +147,7 @@ import org.apache.ignite.internal.processors.service.GridServiceProcessor;
 import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
 import org.apache.ignite.internal.processors.task.GridTaskProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
+import org.apache.ignite.internal.processors.trace.TraceProcessor;
 import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
 import org.apache.ignite.internal.suggestions.JvmConfigurationSuggestions;
 import org.apache.ignite.internal.suggestions.OsConfigurationSuggestions;
@@ -942,6 +943,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
                 startProcessor(new DataStructuresProcessor(ctx));
                 startProcessor(createComponent(PlatformProcessor.class, ctx));
                 startProcessor(new GridMarshallerMappingProcessor(ctx));
+                startProcessor(new TraceProcessor(ctx));
 
                 // Start plugins.
                 for (PluginProvider provider : ctx.plugins().allProviders()) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index bb36b26..62864ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -68,6 +68,7 @@ import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.platform.message.PlatformMessageFilter;
 import org.apache.ignite.internal.processors.pool.PoolProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
+import org.apache.ignite.internal.processors.trace.IgniteTraceAware;
 import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashSet;
 import org.apache.ignite.internal.util.StripedCompositeReadWriteLock;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -982,6 +983,8 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
             // as thread pool may not have any available threads to give.
             byte plc = msg.policy();
 
+            traceNioReceive(msg);
+
             switch (plc) {
                 case P2P_POOL: {
                     processP2PMessage(nodeId, msg, msgC);
@@ -1157,7 +1160,7 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
         catch (RejectedExecutionException e) {
             if (!ctx.isStopping()) {
                 U.error(log, "Failed to process regular message due to execution rejection. Will attempt to process " +
-                        "message in the listener thread instead.", e);
+                    "message in the listener thread instead.", e);
 
                 c.run();
             }
@@ -1167,6 +1170,19 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
     }
 
     /**
+     * @param msg Message to intercept.
+     */
+    private void traceNioReceive(GridIoMessage msg) {
+        if (ctx.trace().tracingEnabled()) {
+            if (msg.message() instanceof IgniteTraceAware) {
+                IgniteTraceAware traceable = (IgniteTraceAware)msg.message();
+
+                traceable.recordTracePoint(IgniteTraceAware.TracePoint.MSG_NIO_RECEIVE);
+            }
+        }
+    }
+
+    /**
      * @param msg Message.
      * @param nodeId Node ID.
      */
@@ -1181,6 +1197,9 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
 
         assert obj != null;
 
+        if (obj instanceof IgniteTraceAware)
+            ((IgniteTraceAware)obj).recordTracePoint(IgniteTraceAware.TracePoint.MSG_LISTENER_INVOKE);
+
         invokeListener(msg.policy(), lsnr, nodeId, obj);
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 29c89a5..b49f79e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -143,6 +143,7 @@ import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQuery
 import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageResponse;
 import org.apache.ignite.internal.processors.rest.handlers.task.GridTaskResultRequest;
 import org.apache.ignite.internal.processors.rest.handlers.task.GridTaskResultResponse;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.GridByteArrayList;
 import org.apache.ignite.internal.util.GridIntList;
 import org.apache.ignite.internal.util.GridLongList;
@@ -179,6 +180,11 @@ public class GridIoMessageFactory implements MessageFactory {
         switch (type) {
             // -54 is reserved for SQL.
             // -46 ... -51 - snapshot messages.
+            case -62:
+                msg = new NodeTrace();
+
+                break;
+
             case -61:
                 msg = new IgniteDiagnosticMessage();
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 9f1873e..07a5bd9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -872,7 +872,8 @@ public class GridCacheIoManager extends GridCacheSharedManagerAdapter {
                     null,
                     null,
                     false,
-                    req.deployInfo() != null);
+                    req.deployInfo() != null,
+                    req.nodeTrace());
 
                 res.error(req.classError());
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java
index ea9336b..7c7b047 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java
@@ -29,6 +29,8 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxState;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxStateAware;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.IgniteTraceAware;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
@@ -41,7 +43,8 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*;
 /**
  * Transaction completion message.
  */
-public class GridDistributedTxFinishRequest extends GridDistributedBaseMessage implements IgniteTxStateAware {
+public class GridDistributedTxFinishRequest extends GridDistributedBaseMessage
+    implements IgniteTxStateAware, IgniteTraceAware {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -109,6 +112,9 @@ public class GridDistributedTxFinishRequest extends GridDistributedBaseMessage i
     @GridDirectTransient
     private IgniteTxState txState;
 
+    /** */
+    protected NodeTrace nodeTrace;
+
     /**
      * Empty constructor required by {@link Externalizable}.
      */
@@ -149,7 +155,8 @@ public class GridDistributedTxFinishRequest extends GridDistributedBaseMessage i
         @Nullable UUID subjId,
         int taskNameHash,
         int txSize,
-        boolean addDepInfo
+        boolean addDepInfo,
+        NodeTrace nodeTrace
     ) {
         super(xidVer, 0, addDepInfo);
 
@@ -169,6 +176,7 @@ public class GridDistributedTxFinishRequest extends GridDistributedBaseMessage i
         this.subjId = subjId;
         this.taskNameHash = taskNameHash;
         this.txSize = txSize;
+        this.nodeTrace = nodeTrace;
 
         completedVersions(committedVers, rolledbackVers);
     }
@@ -306,6 +314,19 @@ public class GridDistributedTxFinishRequest extends GridDistributedBaseMessage i
     }
 
     /** {@inheritDoc} */
+    @Override public void recordTracePoint(TracePoint point) {
+        if (nodeTrace != null)
+            nodeTrace.recordTracePoint(point);
+    }
+
+    /**
+     * @return Message trace, if any.
+     */
+    public NodeTrace nodeTrace() {
+        return nodeTrace;
+    }
+
+    /** {@inheritDoc} */
     @Override public IgniteLogger messageLogger(GridCacheSharedContext ctx) {
         return ctx.txFinishMessageLogger();
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
index c36e633..fbe87fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
@@ -23,6 +23,8 @@ import org.apache.ignite.internal.managers.communication.GridIoMessageFactory;
 import org.apache.ignite.internal.processors.cache.GridCacheMessage;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.IgniteTraceAware;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.lang.IgniteUuid;
@@ -32,7 +34,7 @@ import org.apache.ignite.plugin.extensions.communication.MessageWriter;
 /**
  * Transaction finish response.
  */
-public class GridDistributedTxFinishResponse extends GridCacheMessage {
+public class GridDistributedTxFinishResponse extends GridCacheMessage implements IgniteTraceAware {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -49,6 +51,9 @@ public class GridDistributedTxFinishResponse extends GridCacheMessage {
     /** */
     private int part;
 
+    /** */
+    protected NodeTrace nodeTrace;
+
     /**
      * Empty constructor required by {@link GridIoMessageFactory}.
      */
@@ -61,7 +66,7 @@ public class GridDistributedTxFinishResponse extends GridCacheMessage {
      * @param txId Transaction id.
      * @param futId Future ID.
      */
-    public GridDistributedTxFinishResponse(int part, GridCacheVersion txId, IgniteUuid futId) {
+    public GridDistributedTxFinishResponse(int part, GridCacheVersion txId, IgniteUuid futId, NodeTrace nodeTrace) {
         assert txId != null;
         assert futId != null;
 
@@ -81,6 +86,19 @@ public class GridDistributedTxFinishResponse extends GridCacheMessage {
     }
 
     /** {@inheritDoc} */
+    @Override public void recordTracePoint(TracePoint point) {
+        if (nodeTrace != null)
+            nodeTrace.recordTracePoint(point);
+    }
+
+    /**
+     * @return Message trace, if any.
+     */
+    public NodeTrace nodeTrace() {
+        return nodeTrace;
+    }
+
+    /** {@inheritDoc} */
     @Override public final int partition() {
         return part;
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java
index 91dcd9e..5a5176e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java
@@ -38,6 +38,8 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxState;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxStateAware;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.IgniteTraceAware;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.UUIDCollectionMessage;
 import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -56,7 +58,8 @@ import org.jetbrains.annotations.Nullable;
  * Transaction prepare request for optimistic and eventually consistent
  * transactions.
  */
-public class GridDistributedTxPrepareRequest extends GridDistributedBaseMessage implements IgniteTxStateAware {
+public class GridDistributedTxPrepareRequest extends GridDistributedBaseMessage
+    implements IgniteTxStateAware, IgniteTraceAware {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -157,6 +160,9 @@ public class GridDistributedTxPrepareRequest extends GridDistributedBaseMessage
     @GridToStringExclude
     private byte flags;
 
+    /** */
+    protected NodeTrace nodeTrace;
+
     /**
      * Required by {@link Externalizable}.
      */
@@ -184,7 +190,8 @@ public class GridDistributedTxPrepareRequest extends GridDistributedBaseMessage
         boolean retVal,
         boolean last,
         boolean onePhaseCommit,
-        boolean addDepInfo
+        boolean addDepInfo,
+        NodeTrace nodeTrace
     ) {
         super(tx.xidVersion(), 0, addDepInfo);
 
@@ -199,6 +206,7 @@ public class GridDistributedTxPrepareRequest extends GridDistributedBaseMessage
         this.reads = reads;
         this.writes = writes;
         this.txNodes = txNodes;
+        this.nodeTrace = nodeTrace;
 
         setFlag(tx.system(), SYSTEM_TX_FLAG_MASK);
         setFlag(retVal, NEED_RETURN_VALUE_FLAG_MASK);
@@ -380,6 +388,19 @@ public class GridDistributedTxPrepareRequest extends GridDistributedBaseMessage
         this.txState = txState;
     }
 
+    /** {@inheritDoc} */
+    @Override public void recordTracePoint(TracePoint point) {
+        if (nodeTrace != null)
+            nodeTrace.recordTracePoint(point);
+    }
+
+    /**
+     * @return Request trace, if any.
+     */
+    public NodeTrace nodeTrace() {
+        return nodeTrace;
+    }
+
     /** {@inheritDoc}
      * @param ctx*/
     @Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java
index 58e9492..4d69222 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java
@@ -26,6 +26,8 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxState;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxStateAware;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.IgniteTraceAware;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -35,7 +37,8 @@ import org.apache.ignite.plugin.extensions.communication.MessageWriter;
 /**
  * Response to prepare request.
  */
-public class GridDistributedTxPrepareResponse extends GridDistributedBaseMessage implements IgniteTxStateAware {
+public class GridDistributedTxPrepareResponse extends GridDistributedBaseMessage
+    implements IgniteTxStateAware, IgniteTraceAware {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -57,6 +60,9 @@ public class GridDistributedTxPrepareResponse extends GridDistributedBaseMessage
     /** */
     protected byte flags;
 
+    /** */
+    protected NodeTrace nodeTrace;
+
     /**
      * Empty constructor (required by {@link Externalizable}).
      */
@@ -81,11 +87,18 @@ public class GridDistributedTxPrepareResponse extends GridDistributedBaseMessage
      * @param err Error.
      * @param addDepInfo Deployment info flag.
      */
-    public GridDistributedTxPrepareResponse(int part, GridCacheVersion xid, Throwable err, boolean addDepInfo) {
+    public GridDistributedTxPrepareResponse(
+        int part,
+        GridCacheVersion xid,
+        Throwable err,
+        boolean addDepInfo,
+        NodeTrace nodeTrace
+    ) {
         super(xid, 0, addDepInfo);
 
         this.part = part;
         this.err = err;
+        this.nodeTrace = nodeTrace;
     }
 
     /**
@@ -143,6 +156,19 @@ public class GridDistributedTxPrepareResponse extends GridDistributedBaseMessage
     }
 
     /** {@inheritDoc} */
+    @Override public void recordTracePoint(TracePoint point) {
+        if (nodeTrace != null)
+            nodeTrace.recordTracePoint(point);
+    }
+
+    /**
+     * @return Message trace, if any.
+     */
+    public NodeTrace nodeTrace() {
+        return nodeTrace;
+    }
+
+    /** {@inheritDoc} */
     @Override public IgniteLogger messageLogger(GridCacheSharedContext ctx) {
         return ctx.txPrepareMessageLogger();
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
index 5311ddc..f985759 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
@@ -32,12 +32,12 @@ import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture;
-import org.apache.ignite.internal.processors.cache.GridCacheFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -56,7 +56,7 @@ import static org.apache.ignite.transactions.TransactionState.COMMITTING;
  *
  */
 public final class GridDhtTxFinishFuture<K, V> extends GridCacheCompoundIdentityFuture<IgniteInternalTx>
-    implements GridCacheFuture<IgniteInternalTx>, IgniteDiagnosticAware {
+    implements IgniteDiagnosticAware {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -97,12 +97,20 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCacheCompoundIdentity
     /** Near mappings. */
     private Map<UUID, GridDistributedTxMapping> nearMap;
 
+    /** */
+    private NodeTrace nodeTrace;
+
     /**
      * @param cctx Context.
      * @param tx Transaction.
      * @param commit Commit flag.
      */
-    public GridDhtTxFinishFuture(GridCacheSharedContext<K, V> cctx, GridDhtTxLocalAdapter tx, boolean commit) {
+    public GridDhtTxFinishFuture(
+        GridCacheSharedContext<K, V> cctx,
+        GridDhtTxLocalAdapter tx,
+        boolean commit,
+        NodeTrace nodeTrace
+    ) {
         super(F.<IgniteInternalTx>identityReducer(tx));
 
         this.cctx = cctx;
@@ -118,6 +126,8 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCacheCompoundIdentity
             msgLog = cctx.txFinishMessageLogger();
             log = U.logger(cctx.kernalContext(), logRef, GridDhtTxFinishFuture.class);
         }
+
+        this.nodeTrace = nodeTrace;
     }
 
     /**
@@ -183,6 +193,8 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCacheCompoundIdentity
         if (!isDone()) {
             boolean found = false;
 
+            tx.collectNodeTrace(nodeId, res.nodeTrace());
+
             for (IgniteInternalFuture<IgniteInternalTx> fut : futures()) {
                 if (isMini(fut)) {
                     MiniFuture f = (MiniFuture)fut;
@@ -347,7 +359,8 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCacheCompoundIdentity
                 tx.taskNameHash(),
                 tx.activeCachesDeploymentEnabled(),
                 false,
-                false);
+                false,
+                nodeTrace != null ? new NodeTrace() : null);
 
             try {
                 cctx.io().send(n, req, tx.ioPolicy());
@@ -450,7 +463,8 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCacheCompoundIdentity
                 tx.activeCachesDeploymentEnabled(),
                 updCntrs,
                 false,
-                false);
+                false,
+                nodeTrace != null ? new NodeTrace() : null);
 
             req.writeVersion(tx.writeVersion() != null ? tx.writeVersion() : tx.xidVersion());
 
@@ -519,7 +533,8 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCacheCompoundIdentity
                     tx.taskNameHash(),
                     tx.activeCachesDeploymentEnabled(),
                     false,
-                    false);
+                    false,
+                    nodeTrace != null ? new NodeTrace() : null);
 
                 req.writeVersion(tx.writeVersion());
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
index 90f3687..2c94021 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
@@ -26,6 +26,7 @@ import org.apache.ignite.internal.GridDirectCollection;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxFinishRequest;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.GridLongList;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -121,7 +122,8 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
         int taskNameHash,
         boolean addDepInfo,
         boolean retVal,
-        boolean waitRemoteTxs
+        boolean waitRemoteTxs,
+        NodeTrace nodeTrace
     ) {
         super(
             xidVer,
@@ -140,7 +142,8 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
             subjId,
             taskNameHash,
             txSize,
-            addDepInfo);
+            addDepInfo,
+            nodeTrace);
 
         assert miniId != 0;
         assert nearNodeId != null;
@@ -154,6 +157,8 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
         needReturnValue(retVal);
         waitRemoteTransactions(waitRemoteTxs);
         systemInvalidate(sysInvalidate);
+
+        recordTracePoint(TracePoint.DHT_FINISH_REQUEST_CREATED);
     }
 
     /**
@@ -206,7 +211,8 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
         boolean addDepInfo,
         Collection<Long> updateIdxs,
         boolean retVal,
-        boolean waitRemoteTxs
+        boolean waitRemoteTxs,
+        NodeTrace nodeTrace
     ) {
         this(nearNodeId,
             futId,
@@ -231,7 +237,8 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
             taskNameHash,
             addDepInfo,
             retVal,
-            waitRemoteTxs);
+            waitRemoteTxs,
+            nodeTrace);
 
         if (updateIdxs != null && !updateIdxs.isEmpty()) {
             partUpdateCnt = new GridLongList(updateIdxs.size());

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java
index 6d717eb..9b81065 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java
@@ -26,6 +26,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheReturn;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxFinishResponse;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
@@ -68,12 +69,20 @@ public class GridDhtTxFinishResponse extends GridDistributedTxFinishResponse {
      * @param futId Future ID.
      * @param miniId Mini future ID.
      */
-    public GridDhtTxFinishResponse(int part, GridCacheVersion xid, IgniteUuid futId, int miniId) {
-        super(part, xid, futId);
+    public GridDhtTxFinishResponse(
+        int part,
+        GridCacheVersion xid,
+        IgniteUuid futId,
+        int miniId,
+        NodeTrace nodeTrace
+    ) {
+        super(part, xid, futId, nodeTrace);
 
         assert miniId != 0;
 
         this.miniId = miniId;
+
+        recordTracePoint(TracePoint.DHT_FINISH_RESPONSE_CREATED);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
index 5b8a7b5..47321a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
@@ -39,6 +39,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
@@ -155,7 +156,8 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
             onePhaseCommit,
             txSize,
             subjId,
-            taskNameHash);
+            taskNameHash,
+            null);
 
         assert nearNodeId != null;
         assert nearFutId != null;
@@ -329,7 +331,8 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
         long msgId,
         int nearMiniId,
         Map<UUID, Collection<UUID>> txNodes,
-        boolean last
+        boolean last,
+        NodeTrace nodeTrace
     ) {
         // In optimistic mode prepare still can be called explicitly from salvageTx.
         GridDhtTxPrepareFuture fut = prepFut;
@@ -367,6 +370,8 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
             return chainOnePhasePrepare(fut);
         }
 
+        this.nodeTrace = nodeTrace;
+
         if (state() != PREPARING) {
             if (!state(PREPARING)) {
                 if (state() == PREPARED && isSystemInvalidate())
@@ -478,7 +483,7 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
         if (log.isDebugEnabled())
             log.debug("Committing dht local tx: " + this);
 
-        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, true);
+        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, true, nodeTrace);
 
         cctx.mvcc().addFuture(fut, fut.futureId());
 
@@ -528,7 +533,7 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
      * @return Rollback future.
      */
     public IgniteInternalFuture<IgniteInternalTx> rollbackDhtLocalAsync() {
-        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, false);
+        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, false, nodeTrace);
 
         cctx.mvcc().addFuture(fut, fut.futureId());
 
@@ -583,7 +588,8 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
                 threadId,
                 nearFinFutId,
                 nearFinMiniId,
-                err);
+                err,
+                nodeTrace);
 
             try {
                 cctx.io().send(nearNodeId, res, ioPolicy());

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
index 86eac42..0215f53 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
@@ -41,6 +41,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.F0;
 import org.apache.ignite.internal.util.GridLeanMap;
 import org.apache.ignite.internal.util.GridLeanSet;
@@ -96,6 +97,9 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
     /** Nodes where transactions were started on lock step. */
     private Set<ClusterNode> lockTxNodes;
 
+    /** */
+    protected NodeTrace nodeTrace;
+
     /**
      * Empty constructor required for {@link Externalizable}.
      */
@@ -130,7 +134,8 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
         boolean onePhaseCommit,
         int txSize,
         @Nullable UUID subjId,
-        int taskNameHash
+        int taskNameHash,
+        NodeTrace nodeTrace
     ) {
         super(
             cctx,
@@ -155,6 +160,8 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
         this.explicitLock = explicitLock;
 
         threadId = Thread.currentThread().getId();
+
+        this.nodeTrace = nodeTrace;
     }
 
     /**
@@ -832,6 +839,29 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
     }
 
     /**
+     * @return Node trace.
+     */
+    public NodeTrace nodeTrace() {
+        return nodeTrace;
+    }
+
+    /**
+     * @param nodeTrace Node trace.
+     */
+    public void nodeTrace(NodeTrace nodeTrace) {
+        this.nodeTrace = nodeTrace;
+    }
+
+    /**
+     * @param rmtNodeId Remote node ID.
+     * @param nodeTrace Node trace to collect.
+     */
+    public void collectNodeTrace(UUID rmtNodeId, NodeTrace nodeTrace) {
+        if (this.nodeTrace != null && nodeTrace != null)
+            this.nodeTrace.addRemoteTrace(rmtNodeId, nodeTrace);
+    }
+
+    /**
      * @param prepFut Prepare future.
      * @return If transaction if finished on prepare step returns future which is completed after transaction finish.
      */

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index 6ed4781..cb71d65 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -66,6 +66,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.dr.GridDrType;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
@@ -856,7 +857,8 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
             prepErr,
             null,
             tx.onePhaseCommit(),
-            tx.activeCachesDeploymentEnabled());
+            tx.activeCachesDeploymentEnabled(),
+            tx.nodeTrace());
 
         if (prepErr == null) {
             if (tx.needReturnValue() || tx.nearOnOriginatingNode() || tx.hasInterceptor())
@@ -1290,7 +1292,8 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
                         tx.taskNameHash(),
                         tx.activeCachesDeploymentEnabled(),
                         tx.storeWriteThrough(),
-                        retVal);
+                        retVal,
+                        tx.nodeTrace() != null ? new NodeTrace() : null);
 
                     int idx = 0;
 
@@ -1403,7 +1406,8 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
                             tx.taskNameHash(),
                             tx.activeCachesDeploymentEnabled(),
                             tx.storeWriteThrough(),
-                            retVal);
+                            retVal,
+                            tx.nodeTrace() != null ? new NodeTrace() : null);
 
                         for (IgniteTxEntry entry : nearMapping.entries()) {
                             if (CU.writes().apply(entry)) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
index d334850..f76b1c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
@@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTx
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.GridLeanMap;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -141,7 +142,9 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
         int taskNameHash,
         boolean addDepInfo,
         boolean storeWriteThrough,
-        boolean retVal) {
+        boolean retVal,
+        NodeTrace nodeTrace
+    ) {
         super(tx,
             timeout,
             null,
@@ -150,7 +153,8 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
             retVal,
             last,
             onePhaseCommit,
-            addDepInfo);
+            addDepInfo,
+            nodeTrace);
 
         assert futId != null;
         assert miniId != 0;
@@ -169,6 +173,10 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
         invalidateNearEntries = new BitSet(dhtWrites == null ? 0 : dhtWrites.size());
 
         nearNodeId = tx.nearNodeId();
+
+        this.nodeTrace = nodeTrace;
+
+        recordTracePoint(TracePoint.DHT_PREPARE_REQUEST_CREATED);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java
index 0c2bf81..d4db8c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java
@@ -34,6 +34,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxPrepareResponse;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -111,14 +112,18 @@ public class GridDhtTxPrepareResponse extends GridDistributedTxPrepareResponse {
         IgniteUuid futId,
         int miniId,
         Throwable err,
-        boolean addDepInfo) {
-        super(part, xid, err, addDepInfo);
+        boolean addDepInfo,
+        NodeTrace nodeTrace
+    ) {
+        super(part, xid, err, addDepInfo, nodeTrace);
 
         assert futId != null;
         assert miniId != 0;
 
         this.futId = futId;
         this.miniId = miniId;
+
+        recordTracePoint(TracePoint.DHT_PREPARE_RESPONSE_CREATED);
     }
 
     /**
@@ -160,7 +165,7 @@ public class GridDhtTxPrepareResponse extends GridDistributedTxPrepareResponse {
      * @param invalidPartsByCacheId Map from cache ID to an array of invalid partitions.
      */
     public void invalidPartitionsByCacheId(Map<Integer, Set<Integer>> invalidPartsByCacheId) {
-        this.invalidParts = CU.convertInvalidPartitions(invalidPartsByCacheId);
+        invalidParts = CU.convertInvalidPartitions(invalidPartsByCacheId);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
index 72ddc67..414d8dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
@@ -40,6 +40,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxMapp
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
@@ -199,6 +200,8 @@ public class GridNearOptimisticSerializableTxPrepareFuture extends GridNearOptim
         if (!isDone()) {
             MiniFuture mini = miniFuture(res.miniId());
 
+            tx.collectNodeTrace(nodeId, res.nodeTrace());
+
             if (mini != null)
                 mini.onResult(res, true);
         }
@@ -553,7 +556,8 @@ public class GridNearOptimisticSerializableTxPrepareFuture extends GridNearOptim
             tx.subjectId(),
             tx.taskNameHash(),
             m.clientFirst(),
-            tx.activeCachesDeploymentEnabled());
+            tx.activeCachesDeploymentEnabled(),
+            cctx.kernalContext().trace().tracingEnabled() ? new NodeTrace() : null);
 
         for (IgniteTxEntry txEntry : writes) {
             if (txEntry.op() == TRANSFORM)

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index edddf7d..30d9197 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -44,6 +44,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.transactions.TxDeadlock;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
@@ -184,6 +185,8 @@ public class GridNearOptimisticTxPrepareFuture extends GridNearOptimisticTxPrepa
     /** {@inheritDoc} */
     @Override public void onResult(UUID nodeId, GridNearTxPrepareResponse res) {
         if (!isDone()) {
+            tx.collectNodeTrace(nodeId, res.nodeTrace());
+
             MiniFuture mini = miniFuture(res.miniId());
 
             if (mini != null) {
@@ -535,7 +538,8 @@ public class GridNearOptimisticTxPrepareFuture extends GridNearOptimisticTxPrepa
                     tx.subjectId(),
                     tx.taskNameHash(),
                     m.clientFirst(),
-                    tx.activeCachesDeploymentEnabled());
+                    tx.activeCachesDeploymentEnabled(),
+                    cctx.kernalContext().trace().tracingEnabled() ? new NodeTrace() : null);
 
                 for (IgniteTxEntry txEntry : m.entries()) {
                     if (txEntry.op() == TRANSFORM)

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index e934319..23bd8d6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartit
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxMapping;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -97,6 +98,8 @@ public class GridNearPessimisticTxPrepareFuture extends GridNearTxPrepareFutureA
         if (!isDone()) {
             assert res.clientRemapVersion() == null : res;
 
+            tx.collectNodeTrace(nodeId, res.nodeTrace());
+
             MiniFuture f = miniFuture(res.miniId());
 
             if (f != null) {
@@ -210,7 +213,8 @@ public class GridNearPessimisticTxPrepareFuture extends GridNearTxPrepareFutureA
             tx.subjectId(),
             tx.taskNameHash(),
             false,
-            tx.activeCachesDeploymentEnabled());
+            tx.activeCachesDeploymentEnabled(),
+            cctx.kernalContext().trace().tracingEnabled() ? new NodeTrace() : null);
 
         for (IgniteTxEntry txEntry : writes) {
             if (txEntry.op() == TRANSFORM)

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index c45eb7b..e44ba67 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -45,6 +45,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFini
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -107,7 +108,11 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
      * @param tx Transaction.
      * @param commit Commit flag.
      */
-    public GridNearTxFinishFuture(GridCacheSharedContext<K, V> cctx, GridNearTxLocal tx, boolean commit) {
+    public GridNearTxFinishFuture(
+        GridCacheSharedContext<K, V> cctx,
+        GridNearTxLocal tx,
+        boolean commit
+    ) {
         super(F.<IgniteInternalTx>identityReducer(tx));
 
         this.cctx = cctx;
@@ -209,6 +214,8 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
                 }
             }
 
+            tx.collectNodeTrace(nodeId, res.nodeTrace());
+
             if (finishFut != null)
                 finishFut.onNearFinishResponse(res);
             else {
@@ -463,13 +470,12 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
                 ClusterNode backup = cctx.discovery().node(backupId);
 
                 // Nothing to do if backup has left the grid.
-                if (backup == null) {
-                    // No-op.
+                if (backup != null) {
+                    if (backup.isLocal())
+                        cctx.tm().removeTxReturn(tx.xidVersion());
+                    else
+                        cctx.tm().sendDeferredAckResponse(backupId, tx.xidVersion());
                 }
-                else if (backup.isLocal())
-                    cctx.tm().removeTxReturn(tx.xidVersion());
-                else
-                    cctx.tm().sendDeferredAckResponse(backupId, tx.xidVersion());
             }
         }
     }
@@ -713,7 +719,8 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
             tx.size(),
             tx.subjectId(),
             tx.taskNameHash(),
-            tx.activeCachesDeploymentEnabled()
+            tx.activeCachesDeploymentEnabled(),
+            cctx.kernalContext().trace().tracingEnabled() ? new NodeTrace() : null
         );
 
         // If this is the primary node for the keys.
@@ -847,7 +854,8 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
             0,
             tx.activeCachesDeploymentEnabled(),
             !waitRemoteTxs && (tx.needReturnValue() && tx.implicit()),
-            waitRemoteTxs);
+            waitRemoteTxs,
+            cctx.kernalContext().trace().tracingEnabled() ? new NodeTrace() : null);
 
         finishReq.checkCommitted(true);
 
@@ -857,7 +865,7 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
     /**
      *
      */
-    private abstract class MinFuture extends GridFutureAdapter<IgniteInternalTx> {
+    private abstract static class MinFuture extends GridFutureAdapter<IgniteInternalTx> {
         /** */
         private final int futId;
 
@@ -916,7 +924,7 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
         }
 
         /** {@inheritDoc} */
-        boolean onNodeLeft(UUID nodeId, boolean discoThread) {
+        @Override boolean onNodeLeft(UUID nodeId, boolean discoThread) {
             if (nodeId.equals(m.primary().id())) {
                 if (msgLog.isDebugEnabled()) {
                     msgLog.debug("Near finish fut, mini future node left [txId=" + tx.nearXidVersion() +

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java
index dc32263..f8736ea 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java
@@ -25,6 +25,7 @@ import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxFinishRequest;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
@@ -87,7 +88,9 @@ public class GridNearTxFinishRequest extends GridDistributedTxFinishRequest {
         int txSize,
         @Nullable UUID subjId,
         int taskNameHash,
-        boolean addDepInfo) {
+        boolean addDepInfo,
+        NodeTrace nodeTrace
+    ) {
         super(
             xidVer,
             futId,
@@ -105,11 +108,14 @@ public class GridNearTxFinishRequest extends GridDistributedTxFinishRequest {
             subjId,
             taskNameHash,
             txSize,
-            addDepInfo
+            addDepInfo,
+            nodeTrace
         );
 
         explicitLock(explicitLock);
         storeEnabled(storeEnabled);
+
+        recordTracePoint(TracePoint.NEAR_FINISH_REQUEST_CREATED);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java
index a1a2b57..6ae1b7a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java
@@ -24,6 +24,7 @@ import org.apache.ignite.internal.GridDirectTransient;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxFinishResponse;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
@@ -71,15 +72,18 @@ public class GridNearTxFinishResponse extends GridDistributedTxFinishResponse {
         long nearThreadId,
         IgniteUuid futId,
         int miniId,
-        @Nullable Throwable err)
-    {
-        super(part, xid, futId);
+        @Nullable Throwable err,
+        NodeTrace nodeTrace
+    ) {
+        super(part, xid, futId, nodeTrace);
 
         assert miniId != 0;
 
         this.nearThreadId = nearThreadId;
         this.miniId = miniId;
         this.err = err;
+
+        recordTracePoint(TracePoint.NEAR_FINISH_RESPONSE_CREATED);
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index 81e5ca8..dd678ce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -67,6 +67,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.transactions.TransactionProxy;
 import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
@@ -224,7 +225,8 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
             false,
             txSize,
             subjId,
-            taskNameHash);
+            taskNameHash,
+            ctx.kernalContext().trace().tracingEnabled() ? new NodeTrace() : null);
 
         mappings = implicitSingle ? new IgniteTxMappingsSingleImpl() : new IgniteTxMappingsImpl();
 
@@ -3386,7 +3388,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
             return new GridFinishedFuture<IgniteInternalTx>(this);
         }
 
-        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, true);
+        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, true, nodeTrace);
 
         cctx.mvcc().addFuture(fut, fut.futureId());
 
@@ -3443,7 +3445,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
         if (log.isDebugEnabled())
             log.debug("Rolling back colocated tx locally: " + this);
 
-        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, false);
+        final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, false, nodeTrace);
 
         cctx.mvcc().addFuture(fut, fut.futureId());
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
index 29c7aad..69ed057 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
@@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxPrepareRequest;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -116,7 +117,8 @@ public class GridNearTxPrepareRequest extends GridDistributedTxPrepareRequest {
         @Nullable UUID subjId,
         int taskNameHash,
         boolean firstClientReq,
-        boolean addDepInfo
+        boolean addDepInfo,
+        NodeTrace nodeTrace
     ) {
         super(tx,
             timeout,
@@ -126,7 +128,8 @@ public class GridNearTxPrepareRequest extends GridDistributedTxPrepareRequest {
             retVal,
             last,
             onePhaseCommit,
-            addDepInfo);
+            addDepInfo,
+            nodeTrace);
 
         assert futId != null;
         assert !firstClientReq || tx.optimistic() : tx;
@@ -136,6 +139,8 @@ public class GridNearTxPrepareRequest extends GridDistributedTxPrepareRequest {
         this.subjId = subjId;
         this.taskNameHash = taskNameHash;
 
+        recordTracePoint(TracePoint.NEAR_PREPARE_REQUEST_CREATED);
+
         setFlag(near, NEAR_FLAG_MASK);
         setFlag(implicitSingle, IMPLICIT_SINGLE_FLAG_MASK);
         setFlag(explicitLock, EXPLICIT_LOCK_FLAG_MASK);

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
index 8162168..d803731 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
@@ -35,6 +35,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxPrepareResponse;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.trace.NodeTrace;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -127,9 +128,10 @@ public class GridNearTxPrepareResponse extends GridDistributedTxPrepareResponse
         Throwable err,
         AffinityTopologyVersion clientRemapVer,
         boolean onePhaseCommit,
-        boolean addDepInfo
+        boolean addDepInfo,
+        NodeTrace nodeTrace
     ) {
-        super(part, xid, err, addDepInfo);
+        super(part, xid, err, addDepInfo, nodeTrace);
 
         assert futId != null;
         assert dhtVer != null;
@@ -141,6 +143,8 @@ public class GridNearTxPrepareResponse extends GridDistributedTxPrepareResponse
         this.retVal = retVal;
         this.clientRemapVer = clientRemapVer;
 
+        recordTracePoint(TracePoint.NEAR_PREPARE_RESPONSE_CREATED);
+
         if (onePhaseCommit)
             flags |= NEAR_PREPARE_ONE_PHASE_COMMIT_FLAG_MASK;
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index 02b9352..6b19db1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -70,6 +70,7 @@ import org.apache.ignite.internal.processors.service.GridServiceProcessor;
 import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
 import org.apache.ignite.internal.processors.task.GridTaskProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
+import org.apache.ignite.internal.processors.trace.TraceProcessor;
 import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
 import org.apache.ignite.internal.util.IgniteExceptionRegistry;
 import org.apache.ignite.internal.util.StripedExecutor;
@@ -350,6 +351,11 @@ public class StandaloneGridKernalContext implements GridKernalContext {
     }
 
     /** {@inheritDoc} */
+    @Override public TraceProcessor trace() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
     @Override public void markSegmented() { }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index c473bfe..6998653 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -250,7 +250,8 @@ public class IgniteTxHandler {
                         e,
                         null,
                         req.onePhaseCommit(),
-                        req.deployInfo() != null);
+                        req.deployInfo() != null,
+                        req.nodeTrace());
                 }
             }
         });
@@ -367,7 +368,8 @@ public class IgniteTxHandler {
                         null,
                         top.topologyVersion(),
                         req.onePhaseCommit(),
-                        req.deployInfo() != null);
+                        req.deployInfo() != null,
+                        req.nodeTrace());
 
                     try {
                         ctx.io().send(nearNodeId, res, req.policy());
@@ -463,7 +465,8 @@ public class IgniteTxHandler {
                 req.messageId(),
                 req.miniId(),
                 req.transactionNodes(),
-                req.last());
+                req.last(),
+                req.nodeTrace());
 
             if (tx.isRollbackOnly() && !tx.commitOnPrepare()) {
                 if (tx.state() != TransactionState.ROLLED_BACK && tx.state() != TransactionState.ROLLING_BACK)
@@ -785,7 +788,8 @@ public class IgniteTxHandler {
                 req.threadId(),
                 req.futureId(),
                 req.miniId(),
-                new IgniteCheckedException("Transaction has been already completed."));
+                new IgniteCheckedException("Transaction has been already completed."),
+                req.nodeTrace());
 
             try {
                 ctx.io().send(nodeId, res, req.policy());
@@ -831,6 +835,7 @@ public class IgniteTxHandler {
             tx.nearFinishFutureId(req.futureId());
             tx.nearFinishMiniId(req.miniId());
             tx.storeEnabled(req.storeEnabled());
+            tx.nodeTrace(req.nodeTrace());
 
             if (req.commit()) {
                 if (!tx.markFinalizing(USER_FINISH)) {
@@ -1008,7 +1013,8 @@ public class IgniteTxHandler {
                 req.futureId(),
                 req.miniId(),
                 e,
-                req.deployInfo() != null);
+                req.deployInfo() != null,
+                req.nodeTrace());
         }
 
         if (req.onePhaseCommit()) {
@@ -1328,7 +1334,8 @@ public class IgniteTxHandler {
                 req.partition(),
                 req.version(),
                 req.futureId(),
-                req.miniId());
+                req.miniId(),
+                req.nodeTrace());
 
             if (req.checkCommitted()) {
                 res.checkCommitted(true);

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/trace/IgniteTraceAware.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/trace/IgniteTraceAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/trace/IgniteTraceAware.java
new file mode 100644
index 0000000..594a4cb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/trace/IgniteTraceAware.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.trace;
+
+/**
+ *
+ */
+public interface IgniteTraceAware {
+    public enum TracePoint {
+        MSG_NIO_RECEIVE,
+        MSG_NIO_SEND,
+        MSG_LISTENER_INVOKE,
+        NEAR_PREPARE_REQUEST_CREATED,
+        NEAR_PREPARE_RESPONSE_CREATED,
+        DHT_PREPARE_REQUEST_CREATED,
+        DHT_PREPARE_RESPONSE_CREATED,
+        NEAR_FINISH_REQUEST_CREATED,
+        NEAR_FINISH_RESPONSE_CREATED,
+        DHT_FINISH_REQUEST_CREATED,
+        DHT_FINISH_RESPONSE_CREATED,
+    }
+
+    /**
+     * @param point Trace point.
+     */
+    public void recordTracePoint(TracePoint point);
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/75037190/modules/core/src/main/java/org/apache/ignite/internal/processors/trace/NodeTrace.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/trace/NodeTrace.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/trace/NodeTrace.java
new file mode 100644
index 0000000..ab709c3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/trace/NodeTrace.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.trace;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ *
+ */
+public class NodeTrace implements Message, IgniteTraceAware {
+    /** */
+    private byte[] traceIds;
+
+    /** */
+    private long[] traceStamps;
+
+    /** */
+    private int numTracePoints;
+
+    /** */
+    private List<UUID> rmtNodes;
+
+    /** */
+    private List<NodeTrace> rmtTraces;
+
+    /**
+     *
+     */
+    public NodeTrace() {
+        traceIds = new byte[4];
+        traceStamps = new long[4];
+    }
+
+    /**
+     * @param point Trace point.
+     */
+    @Override public synchronized void recordTracePoint(TracePoint point) {
+        if (traceIds == null || numTracePoints == traceIds.length)
+            expandTracePoints();
+
+        traceIds[numTracePoints] = (byte)point.ordinal();
+        traceStamps[numTracePoints] = System.nanoTime() / 1000;
+
+        numTracePoints++;
+    }
+
+    /**
+     * @param rmtNode Remote node to collect.
+     * @param rmtTrace Remote trace to collect.
+     */
+    public synchronized void addRemoteTrace(UUID rmtNode, NodeTrace rmtTrace) {
+        if (rmtNodes == null) {
+            assert rmtTraces == null;
+
+            rmtNodes = new ArrayList<>(3);
+            rmtTraces = new ArrayList<>(3);
+        }
+
+        rmtNodes.add(rmtNode);
+        rmtTraces.add(rmtTrace);
+    }
+
+    /** {@inheritDoc} */
+    @Override public short directType() {
+        return -62;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
+    /**
+     *
+     */
+    private void expandTracePoints() {
+        if (traceIds == null) {
+            assert traceStamps == null;
+
+            traceIds = new byte[4];
+            traceStamps = new long[4];
+        }
+        else {
+            byte[] cp = new byte[traceIds.length * 2];
+
+            System.arraycopy(traceIds, 0, cp, 0, traceIds.length);
+
+            traceIds = cp;
+
+            long[] stampCp = new long[traceStamps.length * 2];
+
+            System.arraycopy(traceStamps, 0, stampCp, 0, traceStamps.length);
+
+            traceStamps = stampCp;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        return false;
+    }
+}