You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/04/14 14:27:29 UTC

[1/7] ignite git commit: IGNITE-2466 - Use current NIO back pressure mechanism to limit received messages. Mark them process only when backups acknowledged.

Repository: ignite
Updated Branches:
  refs/heads/ignite-4932 788c3fb6c -> ada0f7300


IGNITE-2466 - Use current NIO back pressure mechanism to limit received messages. Mark them process only when backups acknowledged.

(cherry picked from commit 220db88)


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/960e454b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/960e454b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/960e454b

Branch: refs/heads/ignite-4932
Commit: 960e454b4b68c76ed94ca58a3ee9a9d8e79c2039
Parents: 8362fe7
Author: dkarachentsev <dk...@gridgain.com>
Authored: Mon Apr 10 10:28:15 2017 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Mon Apr 10 12:39:33 2017 +0300

----------------------------------------------------------------------
 .../managers/communication/GridIoManager.java   |  12 +-
 .../dht/atomic/GridDhtAtomicCache.java          |  23 ++-
 .../util/nio/GridNioBackPressureControl.java    |  39 ++++-
 .../util/nio/GridNioMessageTracker.java         |   7 +
 .../CacheAtomicPrimarySyncBackPressureTest.java | 151 +++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java       |   3 +
 6 files changed, 220 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/960e454b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 0b7c790..ff7b8a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -736,7 +736,7 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     GridMessageListener lsnr = listenerGet0(msg.topic());
 
@@ -750,7 +750,7 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
                     invokeListener(msg.policy(), lsnr, nodeId, obj);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
 
                     msgC.run();
                 }
@@ -785,12 +785,12 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     processRegularMessage0(msg, nodeId);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
 
                     msgC.run();
                 }
@@ -1146,12 +1146,12 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
         Runnable c = new Runnable() {
             @Override public void run() {
                 try {
-                    threadProcessingMessage(true);
+                    threadProcessingMessage(true, msgC);
 
                     unwindMessageSet(msgSet0, lsnr);
                 }
                 finally {
-                    threadProcessingMessage(false);
+                    threadProcessingMessage(false, null);
                 }
             }
         };

http://git-wip-us.apache.org/repos/asf/ignite/blob/960e454b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 89bdf94..62a5506 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -84,6 +84,8 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.nio.GridNioBackPressureControl;
+import org.apache.ignite.internal.util.nio.GridNioMessageTracker;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
@@ -92,14 +94,15 @@ import org.apache.ignite.internal.util.typedef.CO;
 import org.apache.ignite.internal.util.typedef.CX1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteOutClosure;
+import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
@@ -110,6 +113,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_ATOMIC_DEFERRED_AC
 import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.UPDATE;
@@ -1904,8 +1908,23 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
                         if (req.writeSynchronizationMode() != FULL_ASYNC)
                             req.cleanup(!node.isLocal());
 
-                        if (dhtFut != null)
+                        if (dhtFut != null) {
+                            if (req.writeSynchronizationMode() == PRIMARY_SYNC && !dhtFut.isDone()) {
+                                final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker();
+
+                                if (tracker != null && tracker instanceof GridNioMessageTracker) {
+                                    ((GridNioMessageTracker)tracker).onMessageReceived();
+
+                                    dhtFut.listen(new IgniteInClosure<IgniteInternalFuture<Void>>() {
+                                        @Override public void apply(IgniteInternalFuture<Void> fut) {
+                                            ((GridNioMessageTracker)tracker).onMessageProcessed();
+                                        }
+                                    });
+                                }
+                            }
+
                             ctx.mvcc().addAtomicFuture(dhtFut.version(), dhtFut);
+                        }
                     }
                     else
                         // Should remap all keys.

http://git-wip-us.apache.org/repos/asf/ignite/blob/960e454b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
index 96a1ab3..37d985f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioBackPressureControl.java
@@ -17,14 +17,17 @@
 
 package org.apache.ignite.internal.util.nio;
 
+import org.apache.ignite.lang.IgniteRunnable;
+import org.jetbrains.annotations.Nullable;
+
 /**
  * Utility class that allows to ignore back-pressure control for threads that are processing messages.
  */
 public class GridNioBackPressureControl {
     /** Thread local flag indicating that thread is processing message. */
-    private static ThreadLocal<Boolean> threadProcMsg = new ThreadLocal<Boolean>() {
-        @Override protected Boolean initialValue() {
-            return Boolean.FALSE;
+    private static ThreadLocal<Holder> threadProcMsg = new ThreadLocal<Holder>() {
+        @Override protected Holder initialValue() {
+            return new Holder();
         }
     };
 
@@ -32,13 +35,35 @@ public class GridNioBackPressureControl {
      * @return Flag indicating whether current thread is processing message.
      */
     public static boolean threadProcessingMessage() {
-        return threadProcMsg.get();
+        return threadProcMsg.get().procMsg;
     }
 
     /**
      * @param processing Flag indicating whether current thread is processing message.
+     * @param tracker Thread local back pressure tracker of messages, associated with one connection.
+     */
+    public static void threadProcessingMessage(boolean processing, @Nullable IgniteRunnable tracker) {
+        Holder holder = threadProcMsg.get();
+
+        holder.procMsg = processing;
+        holder.tracker = tracker;
+    }
+
+    /**
+     * @return Thread local back pressure tracker of messages, associated with one connection.
      */
-    public static void threadProcessingMessage(boolean processing) {
-        threadProcMsg.set(processing);
+    @Nullable public static IgniteRunnable threadTracker() {
+        return threadProcMsg.get().tracker;
+    }
+
+    /**
+     *
+     */
+    private static class Holder {
+        /** Process message. */
+        private boolean procMsg;
+
+        /** Tracker. */
+        private IgniteRunnable tracker;
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/960e454b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
index e02c7ca..f05ee0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioMessageTracker.java
@@ -93,6 +93,13 @@ public class GridNioMessageTracker implements IgniteRunnable {
     }
 
     /**
+     *
+     */
+    public void onMessageProcessed() {
+        run();
+    }
+
+    /**
      */
     public void onMessageReceived() {
         int cnt = msgCnt.incrementAndGet();

http://git-wip-us.apache.org/repos/asf/ignite/blob/960e454b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
new file mode 100644
index 0000000..49e3e5c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Checks that back-pressure control restricts uncontrolled growing
+ * of backup message queue. This means, if queue too big - any reads
+ * will be stopped until received acks from backup nodes.
+ */
+public class CacheAtomicPrimarySyncBackPressureTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration ccfg = new CacheConfiguration("cache");
+
+        ccfg.setBackups(1);
+
+        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
+        ccfg.setMemoryMode(CacheMemoryMode.ONHEAP_TIERED);
+        ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+        TestCommunicationSpi spi = new TestCommunicationSpi();
+
+        spi.setMessageQueueLimit(100);
+
+        cfg.setCommunicationSpi(spi);
+        cfg.setClientMode(gridName.contains("client"));
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientPut() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+
+        final Ignite client = startGrid("client");
+
+        checkBackPressure(client, srv1, srv2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testServerPut() throws Exception {
+        Ignite srv1 = startGrid("server1");
+        Ignite srv2 = startGrid("server2");
+
+        final Ignite client = startGrid("server3");
+
+        checkBackPressure(client, srv1, srv2);
+    }
+
+    /**
+     * @param client Producer node.
+     * @throws InterruptedException If failed.
+     */
+    private void checkBackPressure(Ignite client, final Ignite srv1, final Ignite srv2) throws Exception {
+        final IgniteCache<Integer, String> cache = client.cache("cache");
+
+        awaitPartitionMapExchange();
+
+        for (int i = 0; i < 10000; i++) {
+            cache.put(i, String.valueOf(i));
+
+            if (i % 100 == 0) {
+                int size1 = futuresNum(srv1);
+                int size2 = futuresNum(srv2);
+
+                assert size1 < 150 : size1;
+                assert size2 < 150 : size2;
+            }
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @return Size of the backup queue.
+     */
+    private int futuresNum(Ignite ignite) {
+        return ((IgniteKernal)ignite).context().cache().context().mvcc().atomicFutures().size();
+    }
+
+    /**
+     * Delays backup update acks.
+     */
+    private static class TestCommunicationSpi extends TcpCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (((GridIoMessage)msg).message() instanceof GridDhtAtomicDeferredUpdateResponse)
+                sleep(100);
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+
+    /**
+     * @param millis Millis.
+     */
+    private static void sleep(long millis) {
+        try {
+            Thread.sleep(millis);
+        }
+        catch (InterruptedException e) {
+            throw new IgniteSpiException(e);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/960e454b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 5a09a1c..9fcf31a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -94,6 +94,7 @@ import org.apache.ignite.internal.processors.cache.IgniteStartCacheInTransaction
 import org.apache.ignite.internal.processors.cache.IgniteSystemCacheOnClientTest;
 import org.apache.ignite.internal.processors.cache.MarshallerCacheJobRunNodeRestartTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheAffinityEarlyTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheGetFutureHangsSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheNoValueClassOnServerNodeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheCreatePutMultiNodeSelfTest;
@@ -334,6 +335,8 @@ public class IgniteCacheTestSuite4 extends TestSuite {
 
         suite.addTestSuite(IgniteCacheNearOnlyTxTest.class);
 
+        suite.addTestSuite(CacheAtomicPrimarySyncBackPressureTest.class);
+
         return suite;
     }
 }
\ No newline at end of file


[5/7] ignite git commit: Merge branch 'ignite-1.9.1-p1' into ignite-1.9.2

Posted by sb...@apache.org.
Merge branch 'ignite-1.9.1-p1' into ignite-1.9.2


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/241e9291
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/241e9291
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/241e9291

Branch: refs/heads/ignite-4932
Commit: 241e9291dbe43138be1c027a3d8d1b7e3ed9b4dd
Parents: 0f7ef74 6d12bd4
Author: dkarachentsev <dk...@gridgain.com>
Authored: Fri Apr 14 09:36:10 2017 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Fri Apr 14 09:36:10 2017 +0300

----------------------------------------------------------------------

----------------------------------------------------------------------



[3/7] ignite git commit: IGNITE-4917: Fixed failure when accessing BinaryObjectBuilder field value serialized with OptimizedMarshaller . This closes #1736.

Posted by sb...@apache.org.
IGNITE-4917: Fixed failure when accessing BinaryObjectBuilder field value serialized with OptimizedMarshaller . This closes #1736.

(cherry picked from commit b8e3d1b)


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5fbbc376
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5fbbc376
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5fbbc376

Branch: refs/heads/ignite-4932
Commit: 5fbbc37669bcf0557a98d8ab70c3dbb79ebcd340
Parents: 207ccaa
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Wed Apr 5 15:01:02 2017 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Mon Apr 10 12:40:37 2017 +0300

----------------------------------------------------------------------
 .../ignite/internal/binary/builder/BinaryBuilderReader.java | 8 ++++++++
 .../internal/binary/mutabletest/GridBinaryTestClasses.java  | 9 +++++++++
 2 files changed, 17 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/5fbbc376/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
index baaabd6..02264e3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java
@@ -476,6 +476,14 @@ public class BinaryBuilderReader implements BinaryPositionReadable {
                 return new BinaryPlainBinaryObject(binaryObj);
             }
 
+            case GridBinaryMarshaller.OPTM_MARSH: {
+                final BinaryHeapInputStream bin = BinaryHeapInputStream.create(arr, pos + 1);
+
+                final Object obj = BinaryUtils.doReadOptimized(bin, ctx, U.resolveClassLoader(ctx.configuration()));
+
+                return obj;
+            }
+
             default:
                 throw new BinaryObjectException("Invalid flag value: " + type);
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/5fbbc376/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
index 5ddb87dc..0d4de7f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java
@@ -24,6 +24,7 @@ import java.io.ObjectOutput;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Date;
@@ -142,6 +143,9 @@ public class GridBinaryTestClasses {
         public Integer i_;
 
         /** */
+        public BigInteger bi_;
+
+        /** */
         public Long l_;
 
         /** */
@@ -151,6 +155,9 @@ public class GridBinaryTestClasses {
         public Double d_;
 
         /** */
+        public BigDecimal bd_;
+
+        /** */
         public Character c_;
 
         /** */
@@ -267,9 +274,11 @@ public class GridBinaryTestClasses {
             b_ = 11;
             s_ = 22;
             i_ = 33;
+            bi_ = new BigInteger("33000000000000");
             l_ = 44L;
             f_ = 55f;
             d_ = 66d;
+            bd_ = new BigDecimal("33000000000000.123456789");
             c_ = 'e';
             z_ = true;
 


[4/7] ignite git commit: IGNITE-4200: Added copying of the C++ binaries.

Posted by sb...@apache.org.
IGNITE-4200: Added copying of the C++ binaries.

(cherry picked from commit 8b3860f)


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6d12bd4b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6d12bd4b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6d12bd4b

Branch: refs/heads/ignite-4932
Commit: 6d12bd4b9fbded5e5862b5c3d89b05a5ddd11755
Parents: 5fbbc37
Author: Igor Sapego <is...@gridgain.com>
Authored: Tue Mar 21 17:54:51 2017 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Wed Apr 12 12:53:19 2017 +0300

----------------------------------------------------------------------
 assembly/release-fabric-base.xml | 6 ++++++
 1 file changed, 6 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/6d12bd4b/assembly/release-fabric-base.xml
----------------------------------------------------------------------
diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml
index d7aa0a2..97222c9 100644
--- a/assembly/release-fabric-base.xml
+++ b/assembly/release-fabric-base.xml
@@ -179,6 +179,12 @@
             <outputDirectory>/platforms/cpp/docs</outputDirectory>
         </fileSet>
 
+        <!-- Move CPP binaries. -->
+        <fileSet>
+            <directory>modules/platforms/cpp/bin</directory>
+            <outputDirectory>/platforms/cpp/bin</outputDirectory>
+        </fileSet>
+
         <!-- Other files. -->
         <fileSet>
             <directory>bin</directory>


[6/7] ignite git commit: Added IgniteGetFromComputeBenchmark.

Posted by sb...@apache.org.
Added IgniteGetFromComputeBenchmark.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4518f51f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4518f51f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4518f51f

Branch: refs/heads/ignite-4932
Commit: 4518f51f4c1c29517d518f5e44f5d70d17c51170
Parents: 241e929
Author: sboikov <sb...@gridgain.com>
Authored: Fri Apr 14 16:38:46 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Apr 14 16:38:46 2017 +0300

----------------------------------------------------------------------
 .../cache/IgniteGetFromComputeBenchmark.java    | 167 +++++++++++++++++++
 1 file changed, 167 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/4518f51f/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetFromComputeBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetFromComputeBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetFromComputeBenchmark.java
new file mode 100644
index 0000000..66aec82
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetFromComputeBenchmark.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.yardstick.cache.model.SampleValue;
+import org.yardstickframework.BenchmarkConfiguration;
+
+import static org.yardstickframework.BenchmarkUtils.println;
+
+/**
+ * Benchmark created to verify that slow EntryProcessor does not affect 'get' performance.
+ */
+public class IgniteGetFromComputeBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
+    /** */
+    private static final String CACHE_NAME = "atomic-offheap";
+
+    /** */
+    private IgniteCompute compute;
+
+    /** */
+    private IgniteCache asyncCache;
+
+    /** */
+    private ThreadLocal<IgniteFuture> invokeFut = new ThreadLocal<>();
+
+    /** {@inheritDoc} */
+    @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        if (args.preloadAmount() > args.range())
+            throw new IllegalArgumentException("Preloading amount (\"-pa\", \"--preloadAmount\") " +
+                "must by less then the range (\"-r\", \"--range\").");
+
+        String cacheName = cache().getName();
+
+        println(cfg, "Loading data for cache: " + cacheName);
+
+        long start = System.nanoTime();
+
+        try (IgniteDataStreamer<Object, Object> dataLdr = ignite().dataStreamer(cacheName)) {
+            for (int i = 0; i < args.preloadAmount(); i++) {
+                dataLdr.addData(i, new SampleValue(i));
+
+                if (i % 100000 == 0) {
+                    if (Thread.currentThread().isInterrupted())
+                        break;
+
+                    println("Loaded entries: " + i);
+                }
+            }
+        }
+
+        println(cfg, "Finished populating data [time=" + ((System.nanoTime() - start) / 1_000_000) + "ms, " +
+            "amount=" + args.preloadAmount() + ']');
+
+        compute = ignite().compute();
+
+        asyncCache = cache().withAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        IgniteFuture fut = invokeFut.get();
+
+        if (fut == null || fut.isDone()) {
+            Set<Integer> keys = new TreeSet<>();
+
+            for (int i = 0; i < 3; i++)
+                keys.add(nextRandom(args.range()));
+
+            asyncCache.invokeAll(keys, new SlowEntryProcessor(0));
+
+            invokeFut.set(asyncCache.future());
+        }
+
+        int key = nextRandom(args.range());
+
+        compute.affinityCall(CACHE_NAME, key, new GetClosure(key));
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteCache<Integer, Object> cache() {
+        return ignite().cache(CACHE_NAME);
+    }
+
+    /**
+     *
+     */
+    public static class GetClosure implements IgniteCallable<Object> {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** */
+        private final int key;
+
+        /**
+         * @param key Key.
+         */
+        public GetClosure(int key) {
+            this.key = key;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object call() throws Exception {
+            return ignite.cache(CACHE_NAME).get(key);
+        }
+    }
+
+    /**
+     *
+     */
+    public static class SlowEntryProcessor implements CacheEntryProcessor<Integer, Object, Object> {
+        /** */
+        private Object val;
+
+        /**
+         * @param val Value.
+         */
+        public SlowEntryProcessor(Object val) {
+            this.val = val;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object process(MutableEntry<Integer, Object> entry, Object... args) {
+            try {
+                Thread.sleep(10);
+            }
+            catch (InterruptedException ignore) {
+                // No-op.
+            }
+
+            entry.setValue(val);
+
+            return null;
+        }
+    }
+}


[2/7] ignite git commit: IGNITE-2466 - Disable back-pressure for sender data nodes to avoid deadlock.

Posted by sb...@apache.org.
IGNITE-2466 - Disable back-pressure for sender data nodes to avoid deadlock.

(cherry picked from commit ba6227b)

(cherry picked from commit bb3ff12)


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/207ccaa1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/207ccaa1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/207ccaa1

Branch: refs/heads/ignite-4932
Commit: 207ccaa1ff5a021a25dc779b5b2d7bfdb0f3b647
Parents: 960e454
Author: dkarachentsev <dk...@gridgain.com>
Authored: Mon Apr 10 11:40:17 2017 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Mon Apr 10 12:39:44 2017 +0300

----------------------------------------------------------------------
 .../distributed/dht/atomic/GridDhtAtomicCache.java  |  5 ++++-
 .../CacheAtomicPrimarySyncBackPressureTest.java     | 16 ++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/207ccaa1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 62a5506..da16e74 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1909,7 +1909,10 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
                             req.cleanup(!node.isLocal());
 
                         if (dhtFut != null) {
-                            if (req.writeSynchronizationMode() == PRIMARY_SYNC && !dhtFut.isDone()) {
+                            if (req.writeSynchronizationMode() == PRIMARY_SYNC
+                                // To avoid deadlock disable back-pressure for sender data node.
+                                && !ctx.discovery().cacheAffinityNode(ctx.discovery().node(nodeId), ctx.name())
+                                && !dhtFut.isDone()) {
                                 final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker();
 
                                 if (tracker != null && tracker instanceof GridNioMessageTracker) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/207ccaa1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
index 49e3e5c..30436f5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheAtomicPrimarySyncBackPressureTest.java
@@ -30,6 +30,7 @@ import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicDeferredUpdateResponse;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
@@ -53,6 +54,8 @@ public class CacheAtomicPrimarySyncBackPressureTest extends GridCommonAbstractTe
         ccfg.setMemoryMode(CacheMemoryMode.ONHEAP_TIERED);
         ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
 
+        ccfg.setNodeFilter(new NodeFilter());
+
         TestCommunicationSpi spi = new TestCommunicationSpi();
 
         spi.setMessageQueueLimit(100);
@@ -148,4 +151,17 @@ public class CacheAtomicPrimarySyncBackPressureTest extends GridCommonAbstractTe
             throw new IgniteSpiException(e);
         }
     }
+
+    /**
+     * Filters out server node producer.
+     */
+    private static class NodeFilter implements IgnitePredicate<ClusterNode> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode node) {
+            return !("server3".equals(node.attribute("org.apache.ignite.ignite.name")));
+        }
+    }
 }


[7/7] ignite git commit: Merge remote-tracking branch 'remotes/community/ignite-1.9.2' into ignite-4932

Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/community/ignite-1.9.2' into ignite-4932


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ada0f730
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ada0f730
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ada0f730

Branch: refs/heads/ignite-4932
Commit: ada0f7300baa927770ea829207f8e39ba89dcc96
Parents: 788c3fb 4518f51
Author: sboikov <sb...@gridgain.com>
Authored: Fri Apr 14 17:20:32 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Apr 14 17:20:32 2017 +0300

----------------------------------------------------------------------

----------------------------------------------------------------------