You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/03/21 14:01:16 UTC

ignite git commit: debug

Repository: ignite
Updated Branches:
  refs/heads/ignite-3477-debug [created] 05f3aadc8


debug


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/05f3aadc
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/05f3aadc
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/05f3aadc

Branch: refs/heads/ignite-3477-debug
Commit: 05f3aadc8bf0d97d3a27a3a9924eddf010a27f79
Parents: c497388
Author: sboikov <sb...@gridgain.com>
Authored: Tue Mar 21 17:01:06 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Mar 21 17:01:06 2017 +0300

----------------------------------------------------------------------
 .../processors/cache/GridCacheMapEntry.java     |  18 +-
 .../cache/transactions/TestDebugLog.java        | 468 +++++++++++++++++++
 ...idAbstractCacheInterceptorRebalanceTest.java |  27 +-
 .../testsuites/IgniteCacheTestSuite3.java       | 123 +----
 4 files changed, 520 insertions(+), 116 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/05f3aadc/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index d28ea25..395d82d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -56,6 +56,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter;
+import org.apache.ignite.internal.processors.cache.transactions.TestDebugLog;
 import org.apache.ignite.internal.processors.cache.version.GridCacheLazyPlainVersionedEntry;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
@@ -1569,6 +1570,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
         @Nullable final Long updateCntr,
         @Nullable final GridDhtAtomicAbstractUpdateFuture fut
     ) throws IgniteCheckedException, GridCacheEntryRemovedException, GridClosureException {
+        TestDebugLog.addEntryMessage(key.value(cctx.cacheObjectContext(), false), val, "innerUpdate");
+
         assert cctx.atomic() && !detached();
 
         AtomicCacheUpdateClosure c;
@@ -2127,8 +2130,11 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
 
                 obsoleteVersionExtras(obsoleteVer, extras);
 
-                if (clear)
+                if (clear) {
+                    TestDebugLog.addEntryMessage(key.value(cctx.cacheObjectContext(), false), val, "markObsolete0");
+
                     value(null);
+                }
 
                 if (log.isTraceEnabled()) {
                     log.trace("markObsolete0 [key=" + key +
@@ -2525,6 +2531,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
             update |= !preload && deletedUnlocked();
 
             if (update) {
+                TestDebugLog.addEntryMessage(key.value(cctx.cacheObjectContext(), false), val, "initialVal");
+
                 long expTime = expireTime < 0 ? CU.toExpireTime(ttl) : expireTime;
 
                 val = cctx.kernalContext().cacheObjects().prepareForCache(val, cctx);
@@ -2591,6 +2599,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
 
                 return true;
             }
+            else
+                TestDebugLog.addEntryMessage(key.value(cctx.cacheObjectContext(), false), val, "skip initial val");
 
             return false;
         }
@@ -3150,6 +3160,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
         assert Thread.holdsLock(this);
         assert val != null : "null values in update for key: " + key;
 
+        TestDebugLog.addEntryMessage(key.value(cctx.cacheObjectContext(), false), val, "store value");
+
         cctx.offheap().invoke(key,  localPartition(), new UpdateClosure(this, val, ver, expireTime));
     }
 
@@ -3192,6 +3204,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
     protected void removeValue() throws IgniteCheckedException {
         assert Thread.holdsLock(this);
 
+        TestDebugLog.addEntryMessage(key.value(cctx.cacheObjectContext(), false), val, "remove value");
+
         cctx.offheap().remove(key, partition(), localPartition());
     }
 
@@ -3856,6 +3870,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
 
             treeOp = oldRow != null && oldRow.link() == newRow.link() ?
                 IgniteTree.OperationType.NOOP : IgniteTree.OperationType.PUT;
+
+            TestDebugLog.addEntryMessage(entry.key.value(entry.cctx.cacheObjectContext(), false), treeOp, "init treeOp");
         }
 
         /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/05f3aadc/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TestDebugLog.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TestDebugLog.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TestDebugLog.java
new file mode 100644
index 0000000..92abb86
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TestDebugLog.java
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+import java.text.*;
+import java.util.*;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryCustomEventMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryHeartbeatMessage;
+
+/**
+ * TODO
+ if (!cacheMsg.partitionExchangeMessage() && !(cacheMsg instanceof GridDhtPartitionDemandMessage) && !(cacheMsg instanceof GridDhtPartitionSupplyMessage))
+ TestDebugLog.addMessage("Message: " + cacheMsg);
+ */
+public class TestDebugLog {
+    public static volatile boolean DEBUG;
+
+    public static void debugLog(IgniteLogger log, String msg) {
+        if (DEBUG)
+            log.info(msg);
+    }
+
+    /** */
+    private static final List<Object> msgs = Collections.synchronizedList(new ArrayList<>(500_000));
+
+    /** */
+    private static final SimpleDateFormat DEBUG_DATE_FMT = new SimpleDateFormat("HH:mm:ss,SSS");
+
+    static class Message {
+        String thread = Thread.currentThread().getName();
+
+        String msg;
+
+        long ts = U.currentTimeMillis();
+
+        public Message(String msg) {
+            this.msg = msg;
+        }
+
+        public String toString() {
+            return "Msg [thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ", msg=" + msg + ']';
+        }
+    }
+
+    static class PageMessage extends Message {
+        long pageId;
+
+        int bucket;
+
+        public PageMessage(String msg, long pageId, int bucket) {
+            super(msg);
+            this.bucket = bucket;
+            this.pageId = pageId;
+        }
+
+        public String toString() {
+            return "EntryMsg [pageId=" + pageId + ", bucket=" + bucket + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
+        }
+    }
+
+    static class BucketMessage extends Message {
+        int bucket;
+
+        public BucketMessage(String msg, int bucket) {
+            super(msg);
+            this.bucket = bucket;
+        }
+
+        public String toString() {
+            return "BucketMsg [bucket=" + bucket + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
+        }
+    }
+
+    static class EntryMessage extends Message {
+        Object key;
+        Object val;
+
+        public EntryMessage(Object key, Object val, String msg) {
+            super(msg);
+
+            this.key = key;
+            this.val = val;
+        }
+
+        public String toString() {
+            return "EntryMsg [key=" + key + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) +
+                ", val=" + val +']';
+        }
+    }
+    static class PartMessage extends Message {
+        private final int cacheId;
+        private final int partId;
+        private Object val;
+
+        public PartMessage(int cacheId, int partId, Object val, String msg) {
+            super(msg);
+
+            this.cacheId = cacheId;
+            this.partId = partId;
+            this.val = val;
+            this.msg = msg;
+        }
+
+        boolean match(int cacheId, int partId) {
+            return this.cacheId == cacheId && this.partId == partId;
+        }
+
+        @Override public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            PartMessage partKey = (PartMessage) o;
+
+            if (cacheId != partKey.cacheId) return false;
+            return partId == partKey.partId;
+
+        }
+
+        @Override public int hashCode() {
+            int result = cacheId;
+            result = 31 * result + partId;
+            return result;
+        }
+
+        public String toString() {
+            return "PartMessage [partId=" + partId +
+                ", val=" + val +
+                ", msg=" + msg +
+                ", thread=" + thread +
+                ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) +
+                ", cacheId=" + cacheId + ']';
+        }
+    }
+
+    static final boolean out = false;
+
+    public static void addPageMessage(long pageId, String msg) {
+        //msgs.add(new PageMessage(msg, pageId, -1));
+    }
+
+    public static void addBucketMessage(int bucket, String msg) {
+        //msgs.add(new BucketMessage(msg, bucket));
+    }
+
+    public static void addPageMessage(long pageId, int bucket, String msg) {
+        //msgs.add(new PageMessage(msg, pageId, bucket));
+    }
+
+    static final boolean disabled = true;
+
+    public static void addMessage(String msg) {
+        if (disabled)
+            return;
+
+        msgs.add(new Message(msg));
+
+        if (out)
+            System.out.println(msg);
+    }
+
+    public static void addCacheEntryMessage(GridCacheMapEntry e, CacheObject val, String msg) {
+        addEntryMessage(e.key().value(e.context().cacheObjectContext(), false),
+            val != null ? val.value(e.context().cacheObjectContext(), false) : null, msg);
+    }
+
+    public static void addCacheEntryMessage(GridCacheMapEntry e, Object val, String msg) {
+        addEntryMessage(e.key().value(e.context().cacheObjectContext(), false), val, msg);
+    }
+
+    public static void addEntryMessage(Object key, Object val, String msg) {
+//        if (!(val instanceof TcpDiscoveryHeartbeatMessage || val instanceof TcpDiscoveryCustomEventMessage || val instanceof TcpDiscoveryCustomEventMessage))
+//            return;
+
+//        if (val instanceof TcpDiscoveryHeartbeatMessage || val instanceof TcpDiscoveryCustomEventMessage || val instanceof TcpDiscoveryCustomEventMessage)
+//            return;
+//
+        if (disabled)
+            return;
+
+        EntryMessage msg0 = new EntryMessage(key, val, msg);
+
+        msgs.add(msg0);
+
+        if (out)
+            System.out.println(msg0.toString());
+    }
+
+    public static void printMessages(String fileName) {
+        List<Object> msgs0;
+
+        synchronized (msgs) {
+            msgs0 = new ArrayList<>(msgs);
+
+            // msgs.clear();
+        }
+
+        if (fileName != null) {
+            try {
+                FileOutputStream out = new FileOutputStream(fileName);
+
+                PrintWriter w = new PrintWriter(out);
+
+                for (Object msg : msgs0)
+                    w.println(msg.toString());
+
+                w.close();
+
+                out.close();
+            }
+            catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+        else {
+            for (Object msg : msgs0)
+                System.out.println(msg);
+        }
+    }
+
+    public static void addPartMessage(int cacheId,
+        int partId,
+        Object val,
+        String msg) {
+        msgs.add(new PartMessage(cacheId, partId, val, msg));
+    }
+
+    public static void printPartMessages(String fileName, int cacheId, int partId) {
+        List<Object> msgs0;
+
+        synchronized (msgs) {
+            msgs0 = new ArrayList<>(msgs);
+
+            //msgs.clear();
+        }
+
+        if (fileName != null) {
+            try {
+                FileOutputStream out = new FileOutputStream(fileName);
+
+                PrintWriter w = new PrintWriter(out);
+
+                for (Object msg : msgs0) {
+                    if (msg instanceof PartMessage && !((PartMessage)msg).match(cacheId, partId))
+                        continue;
+
+                    w.println(msg.toString());
+                }
+
+                w.close();
+
+                out.close();
+            }
+            catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+        else {
+            for (Object msg : msgs0) {
+                if (msg instanceof PartMessage && !((PartMessage)msg).match(cacheId, partId))
+                    continue;
+
+                System.out.println(msg);
+            }
+        }
+    }
+
+    public static void printPageMessages(String fileName, long pageId) {
+        List<Object> msgs0;
+
+        synchronized (msgs) {
+            msgs0 = new ArrayList<>(msgs);
+
+            //msgs.clear();
+        }
+
+        if (fileName != null) {
+            try {
+                FileOutputStream out = new FileOutputStream(fileName);
+
+                PrintWriter w = new PrintWriter(out);
+
+                for (Object msg : msgs0) {
+                    if (msg instanceof PageMessage && ((PageMessage)msg).pageId != pageId)
+                        continue;
+
+                    w.println(msg.toString());
+                }
+
+                w.close();
+
+                out.close();
+            }
+            catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+        else {
+            for (Object msg : msgs0) {
+                if (msg instanceof PageMessage && ((PageMessage)msg).pageId != pageId)
+                    continue;
+
+                System.out.println(msg);
+            }
+        }
+    }
+
+    public static void printBucketMessages(String fileName, int bucketId) {
+        List<Object> msgs0;
+
+        synchronized (msgs) {
+            msgs0 = new ArrayList<>(msgs);
+
+            //msgs.clear();
+        }
+
+        try {
+            FileOutputStream out = new FileOutputStream(fileName);
+
+            PrintWriter w = new PrintWriter(out);
+
+            for (Object msg : msgs0) {
+                if (msg instanceof BucketMessage && ((BucketMessage)msg).bucket != bucketId)
+                    continue;
+
+                w.println(msg.toString());
+            }
+
+            w.close();
+
+            out.close();
+        }
+        catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public static void printKeyMessages(String fileName, Object key) {
+        printKeyMessages(fileName, Collections.singleton(key));
+    }
+
+    public static void printKeyMessages(String fileName, Set<Object> keys) {
+        assert !F.isEmpty(keys);
+
+        List<Object> msgs0;
+
+        synchronized (msgs) {
+            msgs0 = new ArrayList<>(msgs);
+
+            //msgs.clear();
+        }
+
+        if (fileName != null) {
+            try {
+                FileOutputStream out = new FileOutputStream(fileName);
+
+                PrintWriter w = new PrintWriter(out);
+
+                for (Object msg : msgs0) {
+                    if (msg instanceof EntryMessage && !(keys.contains(((EntryMessage)msg).key)))
+                        continue;
+
+                    w.println(msg.toString());
+                }
+
+                w.close();
+
+                out.close();
+            }
+            catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+        else {
+            for (Object msg : msgs0) {
+                if (msg instanceof EntryMessage && !(keys.contains(((EntryMessage)msg).key)))
+                    continue;
+
+                EntryMessage msg0 = (EntryMessage)msg;
+
+                if (msg0.val instanceof Exception) {
+                    System.out.print(msg0.thread + ", msg=" + msg0.msg + ", ");
+
+                    ((Exception)msg0.val).printStackTrace(System.out);
+                }
+                else
+                    System.out.println(msg);
+            }
+        }
+    }
+
+    public static void printKeyAndPartMessages(String fileName, Object key, int partId, int cacheId) {
+        assert key != null;
+
+        List<Object> msgs0;
+
+        synchronized (msgs) {
+            msgs0 = new ArrayList<>(msgs);
+
+            //msgs.clear();
+        }
+
+        if (fileName != null) {
+            try {
+                FileOutputStream out = new FileOutputStream(fileName);
+
+                PrintWriter w = new PrintWriter(out);
+
+                for (Object msg : msgs0) {
+                    if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
+                        continue;
+
+                    if (msg instanceof PartMessage) {
+                        PartMessage pm = (PartMessage)msg;
+
+                        if (pm.cacheId != cacheId || pm.partId != partId)
+                            continue;
+                    }
+
+                    w.println(msg.toString());
+                }
+
+                w.close();
+
+                out.close();
+            }
+            catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+        else {
+            for (Object msg : msgs0) {
+                if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
+                    continue;
+
+                System.out.println(msg);
+            }
+        }
+    }
+
+    public static void clear() {
+        msgs.clear();
+    }
+
+    public static void main(String[] args) {
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/05f3aadc/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
index f0b3cf0..7aa79ca 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
@@ -25,6 +25,7 @@ import javax.cache.processor.EntryProcessorException;
 import javax.cache.processor.MutableEntry;
 
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheInterceptor;
 import org.apache.ignite.cache.CacheInterceptorAdapter;
@@ -33,6 +34,7 @@ import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.transactions.TestDebugLog;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -73,6 +75,10 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm
     /** */
     private static CacheInterceptor<Integer, Integer> interceptor;
 
+    static {
+        System.setProperty(IgniteSystemProperties.IGNITE_NO_SHUTDOWN_HOOK, "true");
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
         final IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -118,7 +124,7 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm
     /**
      * @throws Exception If fail.
      */
-    public void testRebalanceUpdate() throws Exception {
+    public void _testRebalanceUpdate() throws Exception {
         interceptor = new RebalanceUpdateInterceptor();
 
         testRebalance(new Operation() {
@@ -131,7 +137,7 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm
     /**
      * @throws Exception If fail.
      */
-    public void testRebalanceUpdateInvoke() throws Exception {
+    public void _testRebalanceUpdateInvoke() throws Exception {
         interceptor = new RebalanceUpdateInterceptor();
 
         final UpdateEntryProcessor proc = new UpdateEntryProcessor();
@@ -146,7 +152,7 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm
     /**
      * @throws Exception If fail.
      */
-    public void testRebalanceRemoveInvoke() throws Exception {
+    public void _testRebalanceRemoveInvoke() throws Exception {
         interceptor = new RebalanceUpdateInterceptor();
 
         final RemoveEntryProcessor proc = new RemoveEntryProcessor();
@@ -161,7 +167,7 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm
     /**
      * @throws Exception If fail.
      */
-    public void testRebalanceRemove() throws Exception {
+    public void _testRebalanceRemove() throws Exception {
         interceptor = new RebalanceRemoveInterceptor();
 
         testRebalance(new Operation() {
@@ -174,7 +180,7 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm
     /**
      * @throws Exception If fail.
      */
-    public void testPutIfAbsent() throws Exception {
+    public void _testPutIfAbsent() throws Exception {
         interceptor = new RebalanceUpdateInterceptor();
 
         testRebalance(new Operation() {
@@ -192,6 +198,8 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm
 
         testRebalance(new Operation() {
             @Override public void run(final IgniteCache<Integer, Integer> cache, final Integer key, final Integer val) {
+                TestDebugLog.addEntryMessage(key, val, "start put");
+
                 final Integer old = cache.getAndPut(key, val);
 
                 assert val == old + 1 : "Unexpected old value: " + old;
@@ -209,6 +217,8 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm
         for (int iter = 0; iter < TEST_ITERATIONS && System.currentTimeMillis() < stopTime; iter++) {
             log.info("Iteration: " + iter);
 
+            TestDebugLog.clear();
+
             failed = false;
 
             final IgniteEx ignite = startGrid(1);
@@ -326,6 +336,13 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm
                 else {
                     Integer old = (Integer)entry.getValue();
 
+                    if (old == null) {
+                        TestDebugLog.addEntryMessage(entry.getKey(), null, "null old value");
+                        TestDebugLog.printKeyMessages(null, entry.getKey());
+
+                        System.exit(77);
+                    }
+
                     assertNotNull("Null old value: " + entry, old);
                     assertEquals("Unexpected old value: " + entry, newVal.intValue(), old + 1);
                 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/05f3aadc/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index 4b5e2f6..b1906ea 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -25,6 +25,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheDeploymentOffHeapSel
 import org.apache.ignite.internal.processors.cache.GridCacheDeploymentOffHeapValuesSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheDeploymentSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryVersionSelfTest;
+import org.apache.ignite.internal.processors.cache.GridCacheInterceptorAtomicRebalanceTest;
 import org.apache.ignite.internal.processors.cache.GridCacheOrderedPreloadingSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheReferenceCleanupSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheReloadSelfTest;
@@ -98,116 +99,18 @@ public class IgniteCacheTestSuite3 extends TestSuite {
         TestSuite suite = new TestSuite("IgniteCache Test Suite part 3");
 
         // Value consistency tests.
-        suite.addTestSuite(GridCacheValueConsistencyAtomicSelfTest.class);
-        suite.addTestSuite(GridCacheValueConsistencyAtomicPrimaryWriteOrderSelfTest.class);
-        suite.addTestSuite(GridCacheValueConsistencyAtomicNearEnabledSelfTest.class);
-        suite.addTestSuite(GridCacheValueConsistencyAtomicPrimaryWriteOrderNearEnabledSelfTest.class);
-        suite.addTestSuite(GridCacheValueConsistencyTransactionalSelfTest.class);
-        suite.addTestSuite(GridCacheValueConsistencyTransactionalNearEnabledSelfTest.class);
-        suite.addTestSuite(GridCacheValueBytesPreloadingSelfTest.class);
-
-        // Replicated cache.
-        suite.addTestSuite(GridCacheReplicatedBasicApiTest.class);
-        suite.addTestSuite(GridCacheReplicatedBasicOpSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedBasicStoreSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedGetAndTransformStoreSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedAtomicGetAndTransformStoreSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedEventSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedSynchronousCommitTest.class);
-
-        suite.addTestSuite(GridCacheReplicatedLockSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedMultiNodeLockSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedMultiNodeSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedNodeFailureSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedTxSingleThreadedSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedTxTimeoutSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedPreloadSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedPreloadOffHeapSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedPreloadLifecycleSelfTest.class);
-        suite.addTestSuite(GridCacheSyncReplicatedPreloadSelfTest.class);
-
-        // TODO GG-11141.
-//        suite.addTestSuite(GridCacheDeploymentSelfTest.class);
-//        suite.addTestSuite(GridCacheDeploymentOffHeapSelfTest.class);
-//        suite.addTestSuite(GridCacheDeploymentOffHeapValuesSelfTest.class);
-        suite.addTestSuite(CacheStartupInDeploymentModesTest.class);
-        suite.addTestSuite(GridCacheConditionalDeploymentSelfTest.class);
-        suite.addTestSuite(GridCacheAtomicEntryProcessorDeploymentSelfTest.class);
-        suite.addTestSuite(GridCacheTransactionalEntryProcessorDeploymentSelfTest.class);
-        suite.addTestSuite(IgniteCacheScanPredicateDeploymentSelfTest.class);
-
-        suite.addTestSuite(GridCachePutArrayValueSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedUnswapAdvancedSelfTest.class);
-        // TODO TODO GG-11140.
-        // suite.addTestSuite(GridCacheReplicatedEvictionEventSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedTxMultiThreadedSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedPreloadEventsSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedPreloadStartStopEventsSelfTest.class);
-        suite.addTestSuite(GridReplicatedTxPreloadTest.class);
-
-        suite.addTestSuite(IgniteTxReentryNearSelfTest.class);
-        suite.addTestSuite(IgniteTxReentryColocatedSelfTest.class);
-
-        suite.addTestSuite(GridCacheOrderedPreloadingSelfTest.class);
-        suite.addTestSuite(GridCacheRebalancingSyncSelfTest.class);
-        suite.addTestSuite(GridCacheRebalancingSyncCheckDataTest.class);
-        suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class);
-        suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
-        suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
-
-        // Test for byte array value special case.
-        suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);
-        suite.addTestSuite(GridCacheNearPartitionedP2PEnabledByteArrayValuesSelfTest.class);
-        suite.addTestSuite(GridCacheNearPartitionedP2PDisabledByteArrayValuesSelfTest.class);
-        suite.addTestSuite(GridCachePartitionedOnlyP2PEnabledByteArrayValuesSelfTest.class);
-        suite.addTestSuite(GridCachePartitionedOnlyP2PDisabledByteArrayValuesSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedP2PEnabledByteArrayValuesSelfTest.class);
-        suite.addTestSuite(GridCacheReplicatedP2PDisabledByteArrayValuesSelfTest.class);
-
-        // Near-only cache.
-        suite.addTest(IgniteCacheNearOnlySelfTestSuite.suite());
-
-        // Test cache with daemon nodes.
-        suite.addTestSuite(GridCacheDaemonNodeLocalSelfTest.class);
-        suite.addTestSuite(GridCacheDaemonNodePartitionedSelfTest.class);
-        suite.addTestSuite(GridCacheDaemonNodeReplicatedSelfTest.class);
-
-        // Write-behind.
-        suite.addTest(IgniteCacheWriteBehindTestSuite.suite());
-
-        // Transform.
-        suite.addTestSuite(GridCachePartitionedTransformWriteThroughBatchUpdateSelfTest.class);
-
-        suite.addTestSuite(GridCacheEntryVersionSelfTest.class);
-        suite.addTestSuite(GridCacheVersionSelfTest.class);
-        suite.addTestSuite(GridCacheVersionTopologyChangeTest.class);
-
-        // Memory leak tests.
-        suite.addTestSuite(GridCacheReferenceCleanupSelfTest.class);
-        // TODO GG-11140.
-        // suite.addTestSuite(GridCacheReloadSelfTest.class);
-
-        suite.addTestSuite(GridCacheMixedModeSelfTest.class);
-
-        // Cache metrics.
-        suite.addTest(IgniteCacheMetricsSelfTestSuite.suite());
-
-        // Topology validator.
-        suite.addTest(IgniteTopologyValidatorTestSuit.suite());
-
-        // Eviction.
-        // TODO GG-11140.
-        // suite.addTest(IgniteCacheEvictionSelfTestSuite.suite());
-
-        // Iterators.
-        suite.addTest(IgniteCacheIteratorsSelfTestSuite.suite());
-
-        // Cache interceptor tests.
-        suite.addTest(IgniteCacheInterceptorSelfTestSuite.suite());
-
-        suite.addTestSuite(IgniteTxGetAfterStopTest.class);
-
-        suite.addTestSuite(CacheAsyncOperationsTest.class);
+        suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class);
+        suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class);
+        suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class);
+        suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class);
+        suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class);
+        suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class);
+        suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class);
+        suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class);
+        suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class);
+        suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class);
+        suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class);
+        suite.addTestSuite(GridCacheInterceptorAtomicRebalanceTest.class);
 
         return suite;
     }