You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2018/03/19 06:26:38 UTC

[01/50] [abbrv] ignite git commit: IGNITE-7889: .NET: LINQ: Fix GroupBy with Where

Repository: ignite
Updated Branches:
  refs/heads/ignite-zk a65d955d8 -> 00917728b


IGNITE-7889: .NET: LINQ: Fix GroupBy with Where

This closes #3608


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/64d920bf
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/64d920bf
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/64d920bf

Branch: refs/heads/ignite-zk
Commit: 64d920bf59e18b383f7e2f230f8f4087406e38fe
Parents: a11bcf5
Author: Alexey Popov <ta...@gmail.com>
Authored: Tue Mar 6 20:47:18 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Tue Mar 6 20:47:18 2018 +0300

----------------------------------------------------------------------
 .../Cache/Query/Linq/CacheLinqTest.Functions.cs | 48 ++++++++++++++++++++
 .../Impl/CacheQueryModelVisitor.cs              |  4 ++
 2 files changed, 52 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/64d920bf/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Functions.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Functions.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Functions.cs
index a672d5a..a6632d9 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Functions.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Functions.cs
@@ -110,6 +110,54 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
         }
 
         /// <summary>
+        /// Tests the GroupBy with Where subquery.
+        /// </summary>
+        [Test]
+        public void TestGroupBySubquery()
+        {
+            var persons = GetPersonCache().AsCacheQueryable().Where(p => p.Value.OrganizationId == 1000);
+            var orgs = GetOrgCache().AsCacheQueryable();
+
+            // Simple, unordered
+            CollectionAssert.AreEquivalent(new[] { 1000 },
+                persons.GroupBy(x => x.Value.OrganizationId).Select(x => x.Key).ToArray());
+
+            // Aggregate
+            Assert.AreEqual(1000,
+                persons.GroupBy(x => x.Value.OrganizationId).Select(x => x.Key).OrderBy(x => x).First());
+
+            // Ordering and count
+            var res1 =
+                from p in persons
+                orderby p.Value.Name
+                group p by p.Value.OrganizationId
+                into gs
+                orderby gs.Key
+                where gs.Count() > 10
+                select new { Count = gs.Count(), OrgId = gs.Key, AvgAge = gs.Average(x => x.Value.Age) };
+
+            var resArr = res1.ToArray();
+
+            Assert.AreEqual(new[]
+            {
+                new {Count = PersonCount/2, OrgId = 1000, AvgAge = (double) PersonCount/2 - 1},
+            }, resArr);
+
+            // Join and sum
+            var res2 = persons.Join(orgs.Where(o => o.Key > 10), p => p.Value.OrganizationId, o => o.Key,
+                    (p, o) => new { p, o })
+                .GroupBy(x => x.o.Value.Name)
+                .Select(g => new { Org = g.Key, AgeSum = g.Select(x => x.p.Value.Age).Sum() });
+
+            var resArr2 = res2.ToArray();
+
+            Assert.AreEqual(new[]
+            {
+                new {Org = "Org_0", AgeSum = persons.Where(x => x.Value.OrganizationId == 1000).Sum(x => x.Value.Age)},
+            }, resArr2);
+        }
+
+        /// <summary>
         /// Tests the union.
         /// </summary>
         [Test]

http://git-wip-us.apache.org/repos/asf/ignite/blob/64d920bf/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
index 56c3ff5..223eea6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
@@ -374,6 +374,10 @@ namespace Apache.Ignite.Linq.Impl
             foreach (var join in subQuery.QueryModel.BodyClauses.OfType<JoinClause>())
                 VisitJoinClause(join, queryModel, i++);
 
+            i = 0;
+            foreach (var where in subQuery.QueryModel.BodyClauses.OfType<WhereClause>())
+                VisitWhereClause(where, i++, false);
+
             // Append grouping
             _builder.Append("group by (");
 


[11/50] [abbrv] ignite git commit: IGNITE-7881 Tests for using TreeMap or TreeSet as cache key.

Posted by sb...@apache.org.
IGNITE-7881 Tests for using TreeMap or TreeSet as cache key.

Signed-off-by: Andrey Gura <ag...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/29d56aed
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/29d56aed
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/29d56aed

Branch: refs/heads/ignite-zk
Commit: 29d56aedeae7c00ffe997530aeb7624d5863df5b
Parents: 4ee181c
Author: Ilya Kasnacheev <il...@gmail.com>
Authored: Mon Mar 12 15:19:10 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Mon Mar 12 15:19:10 2018 +0300

----------------------------------------------------------------------
 .../internal/binary/BinaryTreeSelfTest.java     | 273 +++++++++++++++----
 1 file changed, 226 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/29d56aed/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java
index 9481340..819e766 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java
@@ -17,6 +17,11 @@
 
 package org.apache.ignite.internal.binary;
 
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import javax.cache.Cache.Entry;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.binary.BinaryObject;
@@ -29,11 +34,6 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jetbrains.annotations.NotNull;
 
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
 /**
  * Tests for TreeMap and TreeSet structures.
  */
@@ -71,8 +71,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
      *
      * @throws Exception If failed.
      */
-    public void testTreeMapRegularNoComparator() throws Exception {
-        checkTreeMap(false, false);
+    public void testTreeMapAsValueRegularNoComparator() throws Exception {
+        checkTreeMapAsValue(false, false);
     }
 
     /**
@@ -80,8 +80,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
      *
      * @throws Exception If failed.
      */
-    public void testTreeMapRegularComparator() throws Exception {
-        checkTreeMap(false, true);
+    public void testTreeMapAsValueRegularComparator() throws Exception {
+        checkTreeMapAsValue(false, true);
     }
 
     /**
@@ -89,8 +89,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
      *
      * @throws Exception If failed.
      */
-    public void testTreeMapBinaryNoComparator() throws Exception {
-        checkTreeMap(true, false);
+    public void testTreeMapAsValueBinaryNoComparator() throws Exception {
+        checkTreeMapAsValue(true, false);
     }
 
     /**
@@ -98,8 +98,26 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
      *
      * @throws Exception If failed.
      */
-    public void testTreeMapBinaryComparator() throws Exception {
-        checkTreeMap(true, true);
+    public void testTreeMapAsValueBinaryComparator() throws Exception {
+        checkTreeMapAsValue(true, true);
+    }
+
+    /**
+     * Test {@code TreeMap} data structure when used as key.
+     *
+     * @throws Exception If failed.
+     */
+    public void testTreeMapAsKeyNoComparator() throws Exception {
+        checkTreeMapAsKey(false);
+    }
+
+    /**
+     * Test {@code TreeMap} data structure with comparator when used as key.
+     *
+     * @throws Exception If failed.
+     */
+    public void testTreeMapAsKeyComparator() throws Exception {
+        checkTreeMapAsKey(true);
     }
 
     /**
@@ -110,22 +128,11 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     @SuppressWarnings("unchecked")
-    private void checkTreeMap(boolean useBinary, boolean useComparator) throws Exception {
+    private void checkTreeMapAsValue(boolean useBinary, boolean useComparator) throws Exception {
         // Populate map.
         TreeMap<TestKey, Integer> map;
 
-        if (useComparator) {
-            map = new TreeMap<>(new TestKeyComparator());
-
-            for (int i = 0; i < SIZE; i++)
-                map.put(key(false, i), i);
-        }
-        else {
-            map = new TreeMap<>();
-
-            for (int i = 0; i < SIZE; i++)
-                map.put(key(true, i), i);
-        }
+        map = testMap(useComparator);
 
         // Put and get value from cache.
         cache().put(KEY, map);
@@ -147,6 +154,71 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
             assertNull(resMap.comparator());
 
         assertEquals(map, resMap);
+
+        cache().clear();
+    }
+
+    /**
+     * Check {@code TreeMap} data structure when used as key.
+     *
+     * @param useComparator Whether comparator should be used.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("unchecked")
+    private void checkTreeMapAsKey(boolean useComparator) throws Exception {
+        // Populate map.
+        TreeMap<TestKey, Integer> map;
+
+        map = testMap(useComparator);
+
+        // Put and get value from cache.
+        cache().put(map, KEY);
+
+        TreeMap<TestKey, Integer> resMap = (TreeMap<TestKey, Integer>)((Entry)cache().iterator().next()).getKey();
+
+        // Ensure content is correct.
+        if (useComparator)
+            assert resMap.comparator() instanceof TestKeyComparator;
+        else
+            assertNull(resMap.comparator());
+
+        assertEquals(map, resMap);
+
+        // Ensure value is correct.
+        Integer resSameMap = (Integer)cache().get(map);
+
+        assertEquals((Object)KEY, resSameMap);
+
+        Integer resIdenticalMap = (Integer)cache().get(testMap(useComparator));
+
+        assertEquals((Object)KEY, resIdenticalMap);
+
+        // Ensure wrong comparator is not accepted.
+        Integer resDifferentComp = (Integer)cache().get(testMap(!useComparator));
+
+        assertEquals(null, resDifferentComp);
+
+        cache().clear();
+    }
+
+    /** */
+    private TreeMap<TestKey, Integer> testMap(boolean useComp) {
+        TreeMap<TestKey, Integer> map;
+
+        if (useComp) {
+            map = new TreeMap<>(new TestKeyComparator());
+
+            for (int i = 0; i < SIZE; i++)
+                map.put(key(false, i), i);
+        }
+        else {
+            map = new TreeMap<>();
+
+            for (int i = 0; i < SIZE; i++)
+                map.put(key(true, i), i);
+        }
+
+        return map;
     }
 
     /**
@@ -154,8 +226,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
      *
      * @throws Exception If failed.
      */
-    public void testTreeSetRegularNoComparator() throws Exception {
-        checkTreeSet(false, false);
+    public void testTreeSetAsValueRegularNoComparator() throws Exception {
+        checkTreeSetAsValue(false, false);
     }
 
     /**
@@ -163,8 +235,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
      *
      * @throws Exception If failed.
      */
-    public void testTreeSetRegularComparator() throws Exception {
-        checkTreeSet(false, true);
+    public void testTreeSetAsValueRegularComparator() throws Exception {
+        checkTreeSetAsValue(false, true);
     }
 
     /**
@@ -172,8 +244,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
      *
      * @throws Exception If failed.
      */
-    public void testTreeSetBinaryNoComparator() throws Exception {
-        checkTreeSet(true, false);
+    public void testTreeSetAsValueBinaryNoComparator() throws Exception {
+        checkTreeSetAsValue(true, false);
     }
 
     /**
@@ -181,8 +253,26 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
      *
      * @throws Exception If failed.
      */
-    public void testTreeSetBinaryComparator() throws Exception {
-        checkTreeSet(true, true);
+    public void testTreeSetAsValueBinaryComparator() throws Exception {
+        checkTreeSetAsValue(true, true);
+    }
+
+    /**
+     * Test {@code TreeSet} data structure.
+     *
+     * @throws Exception If failed.
+     */
+    public void testTreeSetAsKeyNoComparator() throws Exception {
+        checkTreeSetAsKey(false);
+    }
+
+    /**
+     * Test {@code TreeSet} data structure with comparator.
+     *
+     * @throws Exception If failed.
+     */
+    public void testTreeSetAsKeyComparator() throws Exception {
+        checkTreeSetAsKey(true);
     }
 
     /**
@@ -193,22 +283,11 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     @SuppressWarnings("unchecked")
-    private void checkTreeSet(boolean useBinary, boolean useComparator) throws Exception {
+    private void checkTreeSetAsValue(boolean useBinary, boolean useComparator) throws Exception {
         // Populate set.
         TreeSet<TestKey> set;
 
-        if (useComparator) {
-            set = new TreeSet<>(new TestKeyComparator());
-
-            for (int i = 0; i < SIZE; i++)
-                set.add(key(false, i));
-        }
-        else {
-            set = new TreeSet<>();
-
-            for (int i = 0; i < SIZE; i++)
-                set.add(key(true, i));
-        }
+        set = testSet(useComparator);
 
         // Put and get value from cache.
         cache().put(KEY, set);
@@ -230,6 +309,73 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
             assertNull(resSet.comparator());
 
         assertEquals(set, resSet);
+
+        cache().clear();
+    }
+
+    /**
+     * Check {@code TreeSet} data structure when used as key.
+     *
+     * @param useComp Whether comparator should be used.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("unchecked")
+    private void checkTreeSetAsKey(boolean useComp) throws Exception {
+        // Populate set.
+        TreeSet<TestKey> set;
+
+        set = testSet(useComp);
+
+        // Put and get value from cache.
+        cache().put(set, KEY);
+
+        TreeSet<TestKey> resSet = (TreeSet<TestKey>)((Entry)cache().iterator().next()).getKey();
+
+        // Ensure content is correct.
+        if (useComp)
+            assert resSet.comparator() instanceof TestKeyComparator;
+        else
+            assertNull(resSet.comparator());
+
+        assertEquals(set, resSet);
+
+        // Ensure value is correct.
+        Integer resSameMap = (Integer)cache().get(set);
+
+        assertEquals((Object)KEY, resSameMap);
+
+        Integer resIdenticalMap = (Integer)cache().get(testSet(useComp));
+
+        assertEquals((Object)KEY, resIdenticalMap);
+
+        // Ensure wrong comparator is not accepted.
+        Integer resDifferentComp = (Integer)cache().get(testSet(!useComp));
+
+        assertEquals(null, resDifferentComp);
+
+        assertEquals(set, resSet);
+
+        cache().clear();
+    }
+
+    /** */
+    private TreeSet<TestKey> testSet(boolean useComp) {
+        TreeSet<TestKey> set;
+
+        if (useComp) {
+            set = new TreeSet<>(new TestKeyComparator());
+
+            for (int i = 0; i < SIZE; i++)
+                set.add(key(false, i));
+        }
+        else {
+            set = new TreeSet<>();
+
+            for (int i = 0; i < SIZE; i++)
+                set.add(key(true, i));
+        }
+
+        return set;
     }
 
     /**
@@ -308,6 +454,23 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
         public int id() {
             return id;
         }
+
+        /** */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            TestKey key = (TestKey)o;
+
+            return id == key.id;
+        }
+
+        /** */
+        @Override public int hashCode() {
+            return id;
+        }
     }
 
     /**
@@ -337,5 +500,21 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
         @Override public int compare(TestKey o1, TestKey o2) {
             return o1.id() - o2.id();
         }
+
+        /** */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            return true;
+        }
+
+        /** */
+        @Override public int hashCode() {
+            return 13;
+        }
     }
 }


[17/50] [abbrv] ignite git commit: IGNITE-7860: JDBC thin: changed default socket buffer size to 64Kb. This closes #3600.

Posted by sb...@apache.org.
IGNITE-7860: JDBC thin: changed default socket buffer size to 64Kb. This closes #3600.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1142a368
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1142a368
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1142a368

Branch: refs/heads/ignite-zk
Commit: 1142a368187a07fe91993edebb40bd027c7cb72a
Parents: 3cb9e25
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Tue Mar 13 12:37:29 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Tue Mar 13 12:37:29 2018 +0300

----------------------------------------------------------------------
 .../ignite/jdbc/thin/JdbcThinConnectionSelfTest.java      | 10 ++++++----
 .../internal/jdbc/thin/ConnectionPropertiesImpl.java      |  7 +++++--
 2 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/1142a368/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
index ad98683..bdaa107 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
@@ -155,6 +155,8 @@ public class JdbcThinConnectionSelfTest extends JdbcThinAbstractSelfTest {
      * @throws Exception If failed.
      */
     public void testSocketBuffers() throws Exception {
+        final int dfltDufSize = 64 * 1024;
+
         assertInvalid("jdbc:ignite:thin://127.0.0.1?socketSendBuffer=-1",
             "Property cannot be lower than 0 [name=socketSendBuffer, value=-1]");
 
@@ -162,18 +164,18 @@ public class JdbcThinConnectionSelfTest extends JdbcThinAbstractSelfTest {
             "Property cannot be lower than 0 [name=socketReceiveBuffer, value=-1]");
 
         try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1")) {
-            assertEquals(0, io(conn).connectionProperties().getSocketSendBuffer());
-            assertEquals(0, io(conn).connectionProperties().getSocketReceiveBuffer());
+            assertEquals(dfltDufSize, io(conn).connectionProperties().getSocketSendBuffer());
+            assertEquals(dfltDufSize, io(conn).connectionProperties().getSocketReceiveBuffer());
         }
 
         // Note that SO_* options are hints, so we check that value is equals to either what we set or to default.
         try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?socketSendBuffer=1024")) {
             assertEquals(1024, io(conn).connectionProperties().getSocketSendBuffer());
-            assertEquals(0, io(conn).connectionProperties().getSocketReceiveBuffer());
+            assertEquals(dfltDufSize, io(conn).connectionProperties().getSocketReceiveBuffer());
         }
 
         try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?socketReceiveBuffer=1024")) {
-            assertEquals(0, io(conn).connectionProperties().getSocketSendBuffer());
+            assertEquals(dfltDufSize, io(conn).connectionProperties().getSocketSendBuffer());
             assertEquals(1024, io(conn).connectionProperties().getSocketReceiveBuffer());
         }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/1142a368/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
index 471381b..060c5bf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
@@ -38,6 +38,9 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
     /** Prefix for property names. */
     public static final String PROP_PREFIX = "ignite.jdbc.";
 
+    /** Default socket buffer size. */
+    private static final int DFLT_SOCK_BUFFER_SIZE = 64 * 1024;
+
     /** Host name property. */
     private StringProperty host = new StringProperty(
         "host", "Ignite node IP to connect", null, null, true,
@@ -80,12 +83,12 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
     /** Socket send buffer size property. */
     private IntegerProperty socketSendBuffer = new IntegerProperty(
         "socketSendBuffer", "Socket send buffer size",
-        0, false, 0, Integer.MAX_VALUE);
+        DFLT_SOCK_BUFFER_SIZE, false, 0, Integer.MAX_VALUE);
 
     /** Socket receive buffer size property. */
     private IntegerProperty socketReceiveBuffer = new IntegerProperty(
         "socketReceiveBuffer", "Socket send buffer size",
-        0, false, 0, Integer.MAX_VALUE);
+        DFLT_SOCK_BUFFER_SIZE, false, 0, Integer.MAX_VALUE);
 
     /** Executes update queries on ignite server nodes flag. */
     private BooleanProperty skipReducerOnUpdate = new BooleanProperty(


[46/50] [abbrv] ignite git commit: IGNITE-7964 rmvId is stored to MetaStorage metapage during operations - Fixes #3645.

Posted by sb...@apache.org.
IGNITE-7964 rmvId is stored to MetaStorage metapage during operations - Fixes #3645.

Signed-off-by: Alexey Goncharuk <al...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/892d8c73
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/892d8c73
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/892d8c73

Branch: refs/heads/ignite-zk
Commit: 892d8c739205c2b87598568042f0cffa3004d57b
Parents: f4ac81a
Author: Sergey Chugunov <se...@gmail.com>
Authored: Fri Mar 16 19:24:17 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Mar 16 19:24:17 2018 +0300

----------------------------------------------------------------------
 .../GridCacheDatabaseSharedManager.java         |   4 +-
 .../persistence/metastorage/MetaStorage.java    |  57 +++++--
 .../metastorage/IgniteMetaStorageBasicTest.java | 147 +++++++++++++++++++
 .../ignite/testsuites/IgnitePdsTestSuite.java   |   4 +
 4 files changed, 199 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/892d8c73/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 997f89a..45d9de5 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -553,7 +553,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
             try {
                 restoreMemory(status, true, storePageMem);
 
-                metaStorage = new MetaStorage(cctx.wal(), regCfg, memMetrics, true);
+                metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true);
 
                 metaStorage.init(this);
 
@@ -724,7 +724,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
             cctx.pageStore().initializeForMetastorage();
 
-            metaStorage = new MetaStorage(cctx.wal(), dataRegionMap.get(METASTORE_DATA_REGION_NAME),
+            metaStorage = new MetaStorage(cctx, dataRegionMap.get(METASTORE_DATA_REGION_NAME),
                 (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME));
 
             WALPointer restore = restoreMemory(status);

http://git-wip-us.apache.org/repos/asf/ignite/blob/892d8c73/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index d57b939..d225659 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
@@ -31,6 +32,7 @@ import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.IncompleteObject;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
@@ -79,6 +81,9 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
     private final DataRegion dataRegion;
 
     /** */
+    private final IgniteLogger log;
+
+    /** */
     private MetastorageTree tree;
 
     /** */
@@ -109,17 +114,18 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
     private final Marshaller marshaller = new JdkMarshaller();
 
     /** */
-    public MetaStorage(IgniteWriteAheadLogManager wal, DataRegion dataRegion, DataRegionMetricsImpl regionMetrics,
+    public MetaStorage(GridCacheSharedContext cctx, DataRegion dataRegion, DataRegionMetricsImpl regionMetrics,
         boolean readOnly) {
-        this.wal = wal;
+        wal = cctx.wal();
         this.dataRegion = dataRegion;
         this.regionMetrics = regionMetrics;
         this.readOnly = readOnly;
+        log = cctx.logger(getClass());
     }
 
     /** */
-    public MetaStorage(IgniteWriteAheadLogManager wal, DataRegion memPlc, DataRegionMetricsImpl memMetrics) {
-        this(wal, memPlc, memMetrics, false);
+    public MetaStorage(GridCacheSharedContext cctx, DataRegion memPlc, DataRegionMetricsImpl memMetrics) {
+        this(cctx, memPlc, memMetrics, false);
     }
 
     /** */
@@ -309,6 +315,8 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
 
                     treeRoot = new RootPage(new FullPageId(io.getTreeRoot(pageAddr), METASTORAGE_CACHE_ID), false);
                     reuseListRoot = new RootPage(new FullPageId(io.getReuseListRoot(pageAddr), METASTORAGE_CACHE_ID), false);
+
+                    rmvId.set(io.getGlobalRemoveId(pageAddr));
                 }
                 finally {
                     pageMem.readUnlock(METASTORAGE_CACHE_ID, partId, partMetaPage);
@@ -354,6 +362,8 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
                         treeRoot = io.getTreeRoot(pageAddr);
                         reuseListRoot = io.getReuseListRoot(pageAddr);
 
+                        rmvId.set(io.getGlobalRemoveId(pageAddr));
+
                         assert PageIdUtils.flag(treeRoot) == PageMemory.FLAG_DATA :
                             U.hexLong(treeRoot) + ", part=" + partId + ", METASTORAGE_CACHE_ID=" + METASTORAGE_CACHE_ID;
                         assert PageIdUtils.flag(reuseListRoot) == PageMemory.FLAG_DATA :
@@ -384,21 +394,46 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
     @Override public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
         freeList.saveMetadata();
 
-        MetastorageRowStore rowStore = tree.rowStore();
-
-        saveStoreMetadata(rowStore, ctx);
+        saveStoreMetadata();
     }
 
     /**
-     * @param rowStore Store to save metadata.
      * @throws IgniteCheckedException If failed.
      */
-    private void saveStoreMetadata(MetastorageRowStore rowStore, Context ctx) throws IgniteCheckedException {
-        FreeListImpl freeList = (FreeListImpl)rowStore.freeList();
+    private void saveStoreMetadata() throws IgniteCheckedException {
+        PageMemoryEx pageMem = (PageMemoryEx) pageMemory();
 
-        freeList.saveMetadata();
+        int partId = 0;
+
+        long partMetaId = pageMem.partitionMetaPageId(METASTORAGE_CACHE_ID, partId);
+        long partMetaPage = pageMem.acquirePage(METASTORAGE_CACHE_ID, partMetaId);
+
+        try {
+            long partMetaPageAddr = pageMem.writeLock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+
+            if (partMetaPageAddr == 0L) {
+                U.warn(log, "Failed to acquire write lock for meta page [metaPage=" + partMetaPage + ']');
+
+                return;
+            }
+
+            boolean changed = false;
+
+            try {
+                PagePartitionMetaIO io = PageIO.getPageIO(partMetaPageAddr);
+
+                changed |= io.setGlobalRemoveId(partMetaPageAddr, rmvId.get());
+            }
+            finally {
+                pageMem.writeUnlock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage, null, changed);
+            }
+        }
+        finally {
+            pageMem.releasePage(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+        }
     }
 
+    /** */
     public void applyUpdate(String key, byte[] value) throws IgniteCheckedException {
         if (readOnly) {
             if (lastUpdates == null)

http://git-wip-us.apache.org/repos/asf/ignite/blob/892d8c73/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
new file mode 100644
index 0000000..1837515
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import java.io.Serializable;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ * Single place to add for basic MetaStorage tests.
+ */
+public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        DataStorageConfiguration storageCfg = new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setMaxSize(100 * 1024 * 1024)
+                    .setPersistenceEnabled(true)
+            )
+            .setWalMode(WALMode.LOG_ONLY);
+
+        cfg.setDataStorageConfiguration(storageCfg);
+
+        TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+        discoSpi.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(discoSpi);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+
+        super.beforeTest();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Verifies that MetaStorage after massive amounts of keys stored and updated keys restores its state successfully
+     * after restart.
+     *
+     * See <a hfer="https://issues.apache.org/jira/browse/IGNITE-7964" target="_top">IGNITE-7964</a> for more context
+     * about this test.
+     *
+     * @throws Exception If failed.
+     */
+    public void testMetaStorageMassivePutUpdateRestart() throws Exception {
+        IgniteEx ig = startGrid(0);
+
+        ig.cluster().active(true);
+
+        final byte KEYS_CNT = 100;
+        final String KEY_PREFIX = "test.key.";
+        final String NEW_VAL_PREFIX = "new.val.";
+        final String UPDATED_VAL_PREFIX = "updated.val.";
+
+        loadKeys(ig, KEYS_CNT, KEY_PREFIX, NEW_VAL_PREFIX, UPDATED_VAL_PREFIX);
+
+        stopGrid(0);
+
+        ig = startGrid(0);
+
+        ig.cluster().active(true);
+
+        verifyKeys(ig, KEYS_CNT, KEY_PREFIX, UPDATED_VAL_PREFIX);
+    }
+
+    /** */
+    private void loadKeys(IgniteEx ig,
+        byte keysCnt,
+        String keyPrefix,
+        String newValPrefix,
+        String updatedValPrefix
+    ) throws IgniteCheckedException {
+        IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database();
+
+        MetaStorage metaStorage = db.metaStorage();
+
+        db.checkpointReadLock();
+        try {
+            for (byte i = 0; i < keysCnt; i++)
+                metaStorage.write(keyPrefix + i, newValPrefix + i);
+
+            for (byte i = 0; i < keysCnt; i++)
+                metaStorage.write(keyPrefix + i, updatedValPrefix + i);
+        }
+        finally {
+            db.checkpointReadUnlock();
+        }
+    }
+
+    /** */
+    private void verifyKeys(IgniteEx ig,
+        byte keysCnt,
+        String keyPrefix,
+        String valPrefix
+    ) throws IgniteCheckedException {
+        MetaStorage metaStorage = ig.context().cache().context().database().metaStorage();
+
+        for (byte i = 0; i < keysCnt; i++) {
+            Serializable val = metaStorage.read(keyPrefix + i);
+
+            Assert.assertEquals(valPrefix + i, val);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/892d8c73/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index a36f35b..a9668e7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -26,6 +26,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsDataR
 import org.apache.ignite.internal.processors.cache.persistence.db.file.DefaultPageSizeBackwardsCompatibilityTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCheckpointSimulationWithRealCpDisabledTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsPageReplacementTest;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.IgniteMetaStorageBasicTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreePageMemoryImplTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreeReuseListPageMemoryImplTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.FillFactorMetricTest;
@@ -114,5 +115,8 @@ public class IgnitePdsTestSuite extends TestSuite {
         suite.addTestSuite(IgnitePdsDataRegionMetricsTest.class);
 
         suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
+
+        //MetaStorage
+        suite.addTestSuite(IgniteMetaStorageBasicTest.class);
     }
 }


[25/50] [abbrv] ignite git commit: IGNITE-7253: JDBC thin driver: implemented streaming. This closes #3499. This closes #3591.

Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index f666cdd..bde9427 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -263,7 +263,8 @@ public class GridQueryProcessor extends GridProcessorAdapter {
                     idx.cancelAllQueries();
 
                 return;
-            } catch (InterruptedException ignored) {
+            }
+            catch (InterruptedException ignored) {
                 U.warn(log, "Interrupted while waiting for active queries cancellation.");
 
                 Thread.currentThread().interrupt();
@@ -1974,13 +1975,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
      */
     public List<FieldsQueryCursor<List<?>>> querySqlFields(final SqlFieldsQuery qry, final boolean keepBinary,
         final boolean failOnMultipleStmts) {
-        return querySqlFields(null, qry, keepBinary, failOnMultipleStmts);
-    }
-
-    @SuppressWarnings("unchecked")
-    public FieldsQueryCursor<List<?>> querySqlFields(final GridCacheContext<?,?> cctx, final SqlFieldsQuery qry,
-        final boolean keepBinary) {
-        return querySqlFields(cctx, qry, keepBinary, true).get(0);
+        return querySqlFields(null, qry, null, keepBinary, failOnMultipleStmts);
     }
 
     /**
@@ -1991,7 +1986,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
      * @return Cursor.
      */
     public FieldsQueryCursor<List<?>> querySqlFields(final SqlFieldsQuery qry, final boolean keepBinary) {
-        return querySqlFields(null, qry, keepBinary, true).get(0);
+        return querySqlFields(null, qry, null, keepBinary, true).get(0);
     }
 
     /**
@@ -1999,14 +1994,16 @@ public class GridQueryProcessor extends GridProcessorAdapter {
      *
      * @param cctx Cache context.
      * @param qry Query.
+     * @param cliCtx Client context.
      * @param keepBinary Keep binary flag.
      * @param failOnMultipleStmts If {@code true} the method must throws exception when query contains
      *      more then one SQL statement.
      * @return Cursor.
      */
     @SuppressWarnings("unchecked")
-    public List<FieldsQueryCursor<List<?>>> querySqlFields(@Nullable final GridCacheContext<?,?> cctx,
-        final SqlFieldsQuery qry, final boolean keepBinary, final boolean failOnMultipleStmts) {
+    public List<FieldsQueryCursor<List<?>>> querySqlFields(@Nullable final GridCacheContext<?, ?> cctx,
+        final SqlFieldsQuery qry, final SqlClientContext cliCtx, final boolean keepBinary,
+        final boolean failOnMultipleStmts) {
         checkxEnabled();
 
         validateSqlFieldsQuery(qry);
@@ -2034,7 +2031,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
                     GridQueryCancel cancel = new GridQueryCancel();
 
                     List<FieldsQueryCursor<List<?>>> res =
-                        idx.querySqlFields(schemaName, qry, keepBinary, failOnMultipleStmts, cancel);
+                        idx.querySqlFields(schemaName, qry, cliCtx, keepBinary, failOnMultipleStmts, cancel);
 
                     if (cctx != null)
                         sendQueryExecutedEvent(qry.getSql(), qry.getArgs(), cctx);
@@ -2073,7 +2070,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
      * @param schemaName Schema name.
      * @param streamer Data streamer.
      * @param qry Query.
-     * @return Iterator.
+     * @return Update counter.
      */
     public long streamUpdateQuery(@Nullable final String cacheName, final String schemaName,
         final IgniteDataStreamer<?, ?> streamer, final String qry, final Object[] args) {
@@ -2100,6 +2097,33 @@ public class GridQueryProcessor extends GridProcessorAdapter {
     }
 
     /**
+     * @param schemaName Schema name.
+     * @param cliCtx Client context.
+     * @param qry Query.
+     * @param args Query arguments.
+     * @return Update counters.
+     */
+    public List<Long> streamBatchedUpdateQuery(final String schemaName, final SqlClientContext cliCtx,
+        final String qry, final List<Object[]> args) {
+        if (!busyLock.enterBusy())
+            throw new IllegalStateException("Failed to execute query (grid is stopping).");
+
+        try {
+            return executeQuery(GridCacheQueryType.SQL_FIELDS, qry, null, new IgniteOutClosureX<List<Long>>() {
+                @Override public List<Long> applyx() throws IgniteCheckedException {
+                    return idx.streamBatchedUpdateQuery(schemaName, qry, args, cliCtx);
+                }
+            }, true);
+        }
+        catch (IgniteCheckedException e) {
+            throw new CacheException(e);
+        }
+        finally {
+            busyLock.leaveBusy();
+        }
+    }
+
+    /**
      * Execute distributed SQL query.
      *
      * @param cctx Cache context.

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/query/SqlClientContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/SqlClientContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/SqlClientContext.java
new file mode 100644
index 0000000..e8c2932
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/SqlClientContext.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Container for connection properties passed by various drivers (JDBC drivers, possibly ODBC) having notion of an
+ * <b>SQL connection</b> - Ignite basically does not have one.<p>
+ * Also contains anything that a driver may need to share between threads processing queries of logically same client -
+ * see JDBC thin driver
+ */
+public class SqlClientContext implements AutoCloseable {
+    /** Kernal context. */
+    private final GridKernalContext ctx;
+
+    /** Distributed joins flag. */
+    private final boolean distributedJoins;
+
+    /** Enforce join order flag. */
+    private final boolean enforceJoinOrder;
+
+    /** Collocated flag. */
+    private final boolean collocated;
+
+    /** Replicated caches only flag. */
+    private final boolean replicatedOnly;
+
+    /** Lazy query execution flag. */
+    private final boolean lazy;
+
+    /** Skip reducer on update flag. */
+    private final boolean skipReducerOnUpdate;
+
+    /** Allow overwrites for duplicate keys on streamed {@code INSERT}s. */
+    private boolean streamAllowOverwrite;
+
+    /** Parallel ops count per node for data streamer. */
+    private int streamNodeParOps;
+
+    /** Node buffer size for data streamer. */
+    private int streamNodeBufSize;
+
+    /** Auto flush frequency for streaming. */
+    private long streamFlushTimeout;
+
+    /** Streamers for various caches. */
+    private Map<String, IgniteDataStreamer<?, ?>> streamers;
+
+    /** Logger. */
+    private final IgniteLogger log;
+
+    /**
+     * @param ctx Kernal context.
+     * @param distributedJoins Distributed joins flag.
+     * @param enforceJoinOrder Enforce join order flag.
+     * @param collocated Collocated flag.
+     * @param replicatedOnly Replicated caches only flag.
+     * @param lazy Lazy query execution flag.
+     * @param skipReducerOnUpdate Skip reducer on update flag.
+     */
+    public SqlClientContext(GridKernalContext ctx, boolean distributedJoins, boolean enforceJoinOrder,
+        boolean collocated, boolean replicatedOnly, boolean lazy, boolean skipReducerOnUpdate) {
+        this.ctx = ctx;
+        this.distributedJoins = distributedJoins;
+        this.enforceJoinOrder = enforceJoinOrder;
+        this.collocated = collocated;
+        this.replicatedOnly = replicatedOnly;
+        this.lazy = lazy;
+        this.skipReducerOnUpdate = skipReducerOnUpdate;
+
+        log = ctx.log(SqlClientContext.class.getName());
+    }
+
+    /**
+     * Turn on streaming on this client context.
+     *
+     * @param allowOverwrite Whether streaming should overwrite existing values.
+     * @param flushFreq Flush frequency for streamers.
+     * @param perNodeBufSize Per node streaming buffer size.
+     * @param perNodeParOps Per node streaming parallel operations number.
+     */
+    public void enableStreaming(boolean allowOverwrite, long flushFreq, int perNodeBufSize, int perNodeParOps) {
+        if (isStream())
+            return;
+
+        streamers = new HashMap<>();
+
+        this.streamAllowOverwrite = allowOverwrite;
+        this.streamFlushTimeout = flushFreq;
+        this.streamNodeBufSize = perNodeBufSize;
+        this.streamNodeParOps = perNodeParOps;
+    }
+
+    /**
+     * Turn off streaming on this client context - with closing all open streamers, if any.
+     */
+    public void disableStreaming() {
+        if (!isStream())
+            return;
+
+        Iterator<IgniteDataStreamer<?, ?>> it = streamers.values().iterator();
+
+        while (it.hasNext()) {
+            IgniteDataStreamer<?, ?> streamer = it.next();
+
+            U.close(streamer, log);
+
+            it.remove();
+        }
+
+        streamers = null;
+    }
+
+    /**
+     * @return Collocated flag.
+     */
+    public boolean isCollocated() {
+        return collocated;
+    }
+
+    /**
+     * @return Distributed joins flag.
+     */
+    public boolean isDistributedJoins() {
+        return distributedJoins;
+    }
+
+    /**
+     * @return Enforce join order flag.
+     */
+    public boolean isEnforceJoinOrder() {
+        return enforceJoinOrder;
+    }
+
+    /**
+     * @return Replicated caches only flag.
+     */
+    public boolean isReplicatedOnly() {
+        return replicatedOnly;
+    }
+
+    /**
+     * @return Lazy query execution flag.
+     */
+    public boolean isLazy() {
+        return lazy;
+    }
+
+    /**
+     * @return Skip reducer on update flag,
+     */
+    public boolean isSkipReducerOnUpdate() {
+        return skipReducerOnUpdate;
+    }
+
+    /**
+     * @return Streaming state flag (on or off).
+     */
+    public boolean isStream() {
+        return streamers != null;
+    }
+
+    /**
+     * @param cacheName Cache name.
+     * @return Streamer for given cache.
+     */
+    public IgniteDataStreamer<?, ?> streamerForCache(String cacheName) {
+        if (streamers == null)
+            return null;
+
+        IgniteDataStreamer<?, ?> res = streamers.get(cacheName);
+
+        if (res != null)
+            return res;
+
+        res = ctx.grid().dataStreamer(cacheName);
+
+        res.autoFlushFrequency(streamFlushTimeout);
+
+        res.allowOverwrite(streamAllowOverwrite);
+
+        if (streamNodeBufSize > 0)
+            res.perNodeBufferSize(streamNodeBufSize);
+
+        if (streamNodeParOps > 0)
+            res.perNodeParallelOperations(streamNodeParOps);
+
+        streamers.put(cacheName, res);
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void close() throws Exception {
+        if (streamers == null)
+            return;
+
+        for (IgniteDataStreamer<?, ?> s : streamers.values())
+            U.close(s, log);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java
index 9a2a865..26b2263 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java
@@ -17,11 +17,10 @@
 
 package org.apache.ignite.internal.sql;
 
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.util.typedef.F;
-
 import java.lang.reflect.Field;
 import java.util.HashSet;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.util.typedef.F;
 
 /**
  * SQL keyword constants.
@@ -30,9 +29,15 @@ public class SqlKeyword {
     /** Keyword: ALTER. */
     public static final String ALTER = "ALTER";
 
+    /** Keyword: ALLOW_OVERWRITE */
+    public static final String ALLOW_OVERWRITE = "ALLOW_OVERWRITE";
+
     /** Keyword: ASC. */
     public static final String ASC = "ASC";
 
+    /** Keyword: BATCH_SIZE */
+    public static final String BATCH_SIZE = "BATCH_SIZE";
+
     /** Keyword: BIGINT */
     public static final String BIGINT = "BIGINT";
 
@@ -96,6 +101,9 @@ public class SqlKeyword {
     /** Keyword: FLOAT8. */
     public static final String FLOAT8 = "FLOAT8";
 
+    /** Keyword: FLUSH_FREQUENCY. */
+    public static final String FLUSH_FREQUENCY = "FLUSH_FREQUENCY";
+
     /** Keyword: FORMAT. */
     public static final String FORMAT = "FORMAT";
 
@@ -168,9 +176,18 @@ public class SqlKeyword {
     /** Keyword: NVARCHAR2. */
     public static final String NVARCHAR2 = "NVARCHAR2";
 
+    /** Keyword: OFF. */
+    public static final String OFF = "OFF";
+
     /** Keyword: ON. */
     public static final String ON = "ON";
 
+    /** Keyword: PER_NODE_PARALLEL_OPERATIONS. */
+    public static final String PER_NODE_PARALLEL_OPERATIONS = "PER_NODE_PARALLEL_OPERATIONS";
+
+    /** Keyword: PER_NODE_BUFFER_SIZE. */
+    public static final String PER_NODE_BUFFER_SIZE = "PER_NODE_BUFFER_SIZE";
+
     /** Keyword: PRECISION. */
     public static final String PRECISION = "PRECISION";
 
@@ -183,6 +200,9 @@ public class SqlKeyword {
     /** Keyword: RESTRICT. */
     public static final String RESTRICT = "RESTRICT";
 
+    /** Keyword: SET. */
+    public static final String SET = "SET";
+
     /** Keyword: SIGNED. */
     public static final String SIGNED = "SIGNED";
 
@@ -195,6 +215,9 @@ public class SqlKeyword {
     /** Keyword: SPATIAL. */
     public static final String SPATIAL = "SPATIAL";
 
+    /** Keyword: STREAMING. */
+    public static final String STREAMING = "STREAMING";
+
     /** Keyword: TABLE. */
     public static final String TABLE = "TABLE";
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java
index 0627def..da6d28e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java
@@ -22,6 +22,7 @@ import org.apache.ignite.internal.sql.command.SqlBulkLoadCommand;
 import org.apache.ignite.internal.sql.command.SqlCommand;
 import org.apache.ignite.internal.sql.command.SqlCreateIndexCommand;
 import org.apache.ignite.internal.sql.command.SqlDropIndexCommand;
+import org.apache.ignite.internal.sql.command.SqlSetStreamingCommand;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.internal.sql.SqlKeyword.ALTER;
@@ -31,7 +32,9 @@ import static org.apache.ignite.internal.sql.SqlKeyword.DROP;
 import static org.apache.ignite.internal.sql.SqlKeyword.HASH;
 import static org.apache.ignite.internal.sql.SqlKeyword.INDEX;
 import static org.apache.ignite.internal.sql.SqlKeyword.PRIMARY;
+import static org.apache.ignite.internal.sql.SqlKeyword.SET;
 import static org.apache.ignite.internal.sql.SqlKeyword.SPATIAL;
+import static org.apache.ignite.internal.sql.SqlKeyword.STREAMING;
 import static org.apache.ignite.internal.sql.SqlKeyword.TABLE;
 import static org.apache.ignite.internal.sql.SqlKeyword.UNIQUE;
 import static org.apache.ignite.internal.sql.SqlParserUtils.errorUnexpectedToken;
@@ -110,6 +113,11 @@ public class SqlParser {
 
                             break;
 
+                        case SET:
+                            cmd = processSet();
+
+                            break;
+
                         case ALTER:
                             cmd = processAlter();
                     }
@@ -122,7 +130,7 @@ public class SqlParser {
                         return cmd;
                     }
                     else
-                        throw errorUnexpectedToken(lex, CREATE, DROP, COPY, ALTER);
+                        throw errorUnexpectedToken(lex, CREATE, DROP, ALTER, COPY, SET);
 
                 case QUOTED:
                 case MINUS:
@@ -137,6 +145,22 @@ public class SqlParser {
     }
 
     /**
+     * Process SET keyword.
+     *
+     * @return Command.
+     */
+    private SqlCommand processSet() {
+        if (lex.shift() && lex.tokenType() == SqlLexerTokenType.DEFAULT) {
+            switch (lex.token()) {
+                case STREAMING:
+                    return new SqlSetStreamingCommand().parse(lex);
+            }
+        }
+
+        throw errorUnexpectedToken(lex, STREAMING);
+    }
+
+    /**
      * Processes COPY command.
      *
      * @return The {@link SqlBulkLoadCommand} command.

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParserUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParserUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParserUtils.java
index 829c48c..9ed75ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParserUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParserUtils.java
@@ -123,6 +123,28 @@ public class SqlParserUtils {
     }
 
     /**
+     * Parse boolean parameter value based on presence of tokens 1, 0, ON, OFF. Not that this is not
+     * and is not intended to be routine for parsing a boolean literal from TRUE/FALSE.
+     * @param lex Lexer.
+     * @return Boolean parameter value.
+     */
+    public static boolean parseBoolean(SqlLexer lex) {
+        if (lex.shift() && lex.tokenType() == SqlLexerTokenType.DEFAULT) {
+            switch (lex.token()) {
+                case SqlKeyword.ON:
+                case "1":
+                    return true;
+
+                case SqlKeyword.OFF:
+                case "0":
+                    return false;
+            }
+        }
+
+        throw errorUnexpectedToken(lex, SqlKeyword.ON, SqlKeyword.OFF, "1", "0");
+    }
+
+    /**
      * Process name.
      *
      * @param lex Lexer.

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/sql/command/SqlSetStreamingCommand.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/sql/command/SqlSetStreamingCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/sql/command/SqlSetStreamingCommand.java
new file mode 100644
index 0000000..c492c61
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/sql/command/SqlSetStreamingCommand.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.sql.command;
+
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.sql.SqlKeyword;
+import org.apache.ignite.internal.sql.SqlLexer;
+import org.apache.ignite.internal.sql.SqlLexerTokenType;
+import org.apache.ignite.internal.sql.SqlParseException;
+
+import static org.apache.ignite.internal.sql.SqlParserUtils.error;
+import static org.apache.ignite.internal.sql.SqlParserUtils.errorUnexpectedToken;
+import static org.apache.ignite.internal.sql.SqlParserUtils.parseBoolean;
+import static org.apache.ignite.internal.sql.SqlParserUtils.parseInt;
+
+/**
+ * SET STREAMING command.
+ */
+public class SqlSetStreamingCommand implements SqlCommand {
+    /** Default batch size for driver. */
+    private final static int DFLT_STREAM_BATCH_SIZE = IgniteDataStreamer.DFLT_PER_NODE_BUFFER_SIZE * 4;
+
+    /** Whether streaming must be turned on or off by this command. */
+    private boolean turnOn;
+
+    /** Whether existing values should be overwritten on keys duplication. */
+    private boolean allowOverwrite;
+
+    /** Batch size for driver. */
+    private int batchSize = DFLT_STREAM_BATCH_SIZE;
+
+    /** Per node number of parallel operations. */
+    private int perNodeParOps;
+
+    /** Per node buffer size. */
+    private int perNodeBufSize;
+
+    /** Streamer flush timeout. */
+    private long flushFreq;
+
+    /** {@inheritDoc} */
+    @Override public SqlCommand parse(SqlLexer lex) {
+        turnOn = parseBoolean(lex);
+
+        while (lex.lookAhead().tokenType() == SqlLexerTokenType.DEFAULT) {
+            switch (lex.lookAhead().token()) {
+                case SqlKeyword.BATCH_SIZE:
+                    lex.shift();
+
+                    checkOffLast(lex);
+
+                    batchSize = parseInt(lex);
+
+                    if (batchSize <= 0)
+                        throw error(lex, "Invalid batch size (must be positive).");
+
+                    break;
+
+                case SqlKeyword.PER_NODE_BUFFER_SIZE:
+                    lex.shift();
+
+                    checkOffLast(lex);
+
+                    perNodeBufSize = parseInt(lex);
+
+                    if (perNodeBufSize <= 0)
+                        throw error(lex, "Invalid per node buffer size (must be positive).");
+
+                    break;
+
+                case SqlKeyword.PER_NODE_PARALLEL_OPERATIONS:
+                    lex.shift();
+
+                    checkOffLast(lex);
+
+                    perNodeParOps = parseInt(lex);
+
+                    if (perNodeParOps <= 0)
+                        throw error(lex, "Invalid per node parallel operations number (must be positive).");
+
+                    break;
+
+                case SqlKeyword.ALLOW_OVERWRITE:
+                    lex.shift();
+
+                    checkOffLast(lex);
+
+                    allowOverwrite = parseBoolean(lex);
+
+                    break;
+
+                case SqlKeyword.FLUSH_FREQUENCY:
+                    lex.shift();
+
+                    checkOffLast(lex);
+
+                    flushFreq = parseInt(lex);
+
+                    if (flushFreq <= 0)
+                        throw error(lex, "Invalid flush frequency (must be positive).");
+
+                    break;
+
+                default:
+                    return this;
+            }
+        }
+
+        return this;
+    }
+
+    /**
+     * Throw an unexpected token exception if this command turns streaming off.
+     * @param lex Lexer to take unexpected token from.
+     * @throws SqlParseException if {@link #turnOn} is {@code false}.
+     */
+    private void checkOffLast(SqlLexer lex) throws SqlParseException {
+        if (!turnOn) {
+            assert lex.tokenType() == SqlLexerTokenType.DEFAULT;
+
+            throw errorUnexpectedToken(lex);
+        }
+    }
+
+    /**
+     * @return Whether streaming must be turned on or off by this command.
+     */
+    public boolean isTurnOn() {
+        return turnOn;
+    }
+
+    /**
+     * @return Whether existing values should be overwritten on keys duplication.
+     */
+    public boolean allowOverwrite() {
+        return allowOverwrite;
+    }
+
+    /**
+     * @return Batch size for driver.
+     */
+    public int batchSize() {
+        return batchSize;
+    }
+
+    /**
+     * @return Per node number of parallel operations.
+     */
+    public int perNodeParallelOperations() {
+        return perNodeParOps;
+    }
+
+    /**
+     * @return Per node streamer buffer size.
+     */
+    public int perNodeBufferSize() {
+        return perNodeBufSize;
+    }
+
+    /**
+     * @return Streamer flush timeout
+     */
+    public long flushFrequency() {
+        return flushFreq;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String schemaName() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void schemaName(String schemaName) {
+        // No-op.
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
index df27c5f..6d7e9ae 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
@@ -52,6 +52,7 @@ import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
 import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
 import org.apache.ignite.internal.processors.query.QueryField;
 import org.apache.ignite.internal.processors.query.QueryIndexDescriptorImpl;
+import org.apache.ignite.internal.processors.query.SqlClientContext;
 import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitor;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
 import org.apache.ignite.internal.util.lang.GridCloseableIterator;
@@ -244,12 +245,18 @@ public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractT
 
         /** {@inheritDoc} */
         @Override public List<FieldsQueryCursor<List<?>>> querySqlFields(String schemaName, SqlFieldsQuery qry,
-            boolean keepBinary, boolean failOnMultipleStmts, GridQueryCancel cancel) {
+            SqlClientContext cliCtx, boolean keepBinary, boolean failOnMultipleStmts, GridQueryCancel cancel) {
             return null;
         }
 
         /** {@inheritDoc} */
-        @Override public long streamUpdateQuery(String spaceName, String qry, @Nullable Object[] params,
+        @Override public List<Long> streamBatchedUpdateQuery(String schemaName, String qry, List<Object[]> params,
+            SqlClientContext cliCtx) throws IgniteCheckedException {
+            return Collections.emptyList();
+        }
+
+        /** {@inheritDoc} */
+        @Override public long streamUpdateQuery(String schemaName, String qry, @Nullable Object[] params,
             IgniteDataStreamer<?, ?> streamer) throws IgniteCheckedException {
             return 0;
         }
@@ -372,8 +379,8 @@ public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractT
         }
 
         /** {@inheritDoc} */
-        @Override public boolean isInsertStatement(PreparedStatement nativeStmt) {
-            return false;
+        @Override public void checkStatementStreamable(PreparedStatement nativeStmt) {
+            // No-op.
         }
 
         /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/test/java/org/apache/ignite/internal/sql/SqlParserSetStreamingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/sql/SqlParserSetStreamingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/sql/SqlParserSetStreamingSelfTest.java
new file mode 100644
index 0000000..65bb599
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/sql/SqlParserSetStreamingSelfTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.sql;
+
+import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.sql.command.SqlSetStreamingCommand;
+
+/**
+ * Tests for SQL parser: SET STREAMING.
+ */
+public class SqlParserSetStreamingSelfTest extends SqlParserAbstractSelfTest {
+    /**
+     *
+     */
+    public void testParseSetStreaming() {
+        parseValidate("set streaming on", true, false, 2048, 0, 0, 0);
+        parseValidate("set streaming 1", true, false, 2048, 0, 0, 0);
+        parseValidate("set streaming off", false, false, 2048, 0, 0, 0);
+        parseValidate("set streaming 0", false, false, 2048, 0, 0, 0);
+        parseValidate("set streaming on batch_size 100", true, false, 100, 0, 0, 0);
+        parseValidate("set streaming on flush_frequency 500", true, false, 2048, 0, 0, 500);
+        parseValidate("set streaming on per_node_buffer_size 100", true, false, 2048, 0, 100, 0);
+        parseValidate("set streaming on per_node_parallel_operations 4", true, false, 2048, 4, 0, 0);
+        parseValidate("set streaming on allow_overwrite on", true, true, 2048, 0, 0, 0);
+        parseValidate("set streaming on allow_overwrite off", true, false, 2048, 0, 0, 0);
+        parseValidate("set streaming on per_node_buffer_size 50 flush_frequency 500 " +
+            "per_node_parallel_operations 4 allow_overwrite on batch_size 100", true, true, 100, 4, 50, 500);
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set",
+            "Failed to parse SQL statement \"set[*]\": Unexpected end of command (expected: \"STREAMING\")");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming",
+            "Failed to parse SQL statement \"set streaming[*]\": Unexpected end of command (expected: " +
+                "\"ON\", \"OFF\", \"1\", \"0\")");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming table",
+            "Failed to parse SQL statement \"set streaming [*]table\": Unexpected token: \"TABLE\" (expected: " +
+                "\"ON\", \"OFF\", \"1\", \"0\")");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming -1",
+            "Failed to parse SQL statement \"set streaming [*]-1\": Unexpected token: \"-\" (expected: " +
+                "\"ON\", \"OFF\", \"1\", \"0\")");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming 500",
+            "Failed to parse SQL statement \"set streaming [*]500\": Unexpected token: \"500\" (expected: " +
+                "\"ON\", \"OFF\", \"1\", \"0\")");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming on allow_overwrite",
+            "set streaming on allow_overwrite[*]\": Unexpected end of command (expected: \"ON\", \"OFF\", \"1\", " +
+                "\"0\")");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming 1 batch_size",
+            "Failed to parse SQL statement \"set streaming 1 batch_size[*]\": Unexpected end of command " +
+                "(expected: \"[integer]\")");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming on per_node_parallel_operations -4",
+            "Failed to parse SQL statement \"set streaming on per_node_parallel_operations -[*]4\": " +
+                "Invalid per node parallel operations number (must be positive)");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming on per_node_buffer_size -4",
+            "Failed to parse SQL statement \"set streaming on per_node_buffer_size -[*]4\": " +
+                "Invalid per node buffer size (must be positive)");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming on flush_frequency -4",
+            "Failed to parse SQL statement \"set streaming on flush_frequency -[*]4\": " +
+                "Invalid flush frequency (must be positive)");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming off allow_overwrite",
+            "Failed to parse SQL statement \"set streaming off [*]allow_overwrite\": Unexpected token: " +
+                "\"ALLOW_OVERWRITE\"");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming off batch_size",
+            "Failed to parse SQL statement \"set streaming off [*]batch_size\": Unexpected token: " +
+                "\"BATCH_SIZE\"");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming off flush_frequency",
+            "Failed to parse SQL statement \"set streaming off [*]flush_frequency\": Unexpected token: " +
+                "\"FLUSH_FREQUENCY\"");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming off per_node_buffer_size",
+            "Failed to parse SQL statement \"set streaming off [*]per_node_buffer_size\": Unexpected token: " +
+                "\"PER_NODE_BUFFER_SIZE\"");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming off per_node_parallel_operations",
+            "Failed to parse SQL statement \"set streaming off [*]per_node_parallel_operations\": Unexpected token: " +
+                "\"PER_NODE_PARALLEL_OPERATIONS\"");
+
+        assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming off table",
+            "Failed to parse SQL statement \"set streaming off [*]table\": Unexpected token: \"TABLE\"");
+    }
+
+    /**
+     * Parse and validate SQL script.
+     *
+     * @param sql SQL.
+     * @param expOn Expected on/off value.
+     * @param expAllowOverwrite Expected allow overwrite flag.
+     * @param expBatchSize Expected batch size.
+     * @param expParOps Expected per-node parallael operations.
+     * @param expBufSize Expected per node buffer size.
+     * @param expFlushFreq Expected flush frequency.
+     */
+    private static void parseValidate(String sql, boolean expOn, boolean expAllowOverwrite, int expBatchSize,
+        int expParOps, int expBufSize, long expFlushFreq) {
+        SqlSetStreamingCommand cmd = (SqlSetStreamingCommand)new SqlParser(QueryUtils.DFLT_SCHEMA, sql).nextCommand();
+
+        assertEquals(expOn, cmd.isTurnOn());
+
+        assertEquals(expAllowOverwrite, cmd.allowOverwrite());
+
+        assertEquals(expBatchSize, cmd.batchSize());
+
+        assertEquals(expParOps, cmd.perNodeParallelOperations());
+
+        assertEquals(expBufSize, cmd.perNodeBufferSize());
+
+        assertEquals(expFlushFreq, cmd.flushFrequency());
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index b114828..8d2a820 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -299,7 +299,7 @@ public final class GridTestUtils {
             call.call();
         }
         catch (Throwable e) {
-            if (cls != e.getClass()) {
+            if (cls != e.getClass() && !cls.isAssignableFrom(e.getClass())) {
                 if (e.getClass() == CacheException.class && e.getCause() != null && e.getCause().getClass() == cls)
                     e = e.getCause();
                 else {

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
index ea6c7c1..62dbd50 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
@@ -43,10 +43,10 @@ import org.apache.ignite.cache.query.BulkLoadContextCursor;
 import org.apache.ignite.cache.query.FieldsQueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.processors.bulkload.BulkLoadCacheWriter;
-import org.apache.ignite.internal.processors.bulkload.BulkLoadProcessor;
 import org.apache.ignite.internal.processors.bulkload.BulkLoadAckClientParameters;
+import org.apache.ignite.internal.processors.bulkload.BulkLoadCacheWriter;
 import org.apache.ignite.internal.processors.bulkload.BulkLoadParser;
+import org.apache.ignite.internal.processors.bulkload.BulkLoadProcessor;
 import org.apache.ignite.internal.processors.bulkload.BulkLoadStreamerWriter;
 import org.apache.ignite.internal.processors.cache.CacheOperationContext;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -385,6 +385,7 @@ public class DmlStatementsProcessor {
     /**
      * Perform given statement against given data streamer. Only rows based INSERT is supported.
      *
+     * @param schemaName Schema name.
      * @param streamer Streamer to feed data to.
      * @param stmt Statement.
      * @param args Statement arguments.
@@ -392,81 +393,74 @@ public class DmlStatementsProcessor {
      * @throws IgniteCheckedException if failed.
      */
     @SuppressWarnings({"unchecked", "ConstantConditions"})
-    long streamUpdateQuery(IgniteDataStreamer streamer, PreparedStatement stmt, final Object[] args)
+    long streamUpdateQuery(String schemaName, IgniteDataStreamer streamer, PreparedStatement stmt, final Object[] args)
         throws IgniteCheckedException {
+        idx.checkStatementStreamable(stmt);
+
         Prepared p = GridSqlQueryParser.prepared(stmt);
 
         assert p != null;
 
-        final UpdatePlan plan = UpdatePlanBuilder.planForStatement(p, true, idx, null, null, null);
-
-        if (!F.eq(streamer.cacheName(), plan.cacheContext().name()))
-            throw new IgniteSQLException("Cross cache streaming is not supported, please specify cache explicitly" +
-                " in connection options", IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
-
-        if (plan.mode() == UpdateMode.INSERT && plan.rowCount() > 0) {
-            assert plan.isLocalSubquery();
+        final UpdatePlan plan = getPlanForStatement(schemaName, null, p, null, true, null);
 
-            final GridCacheContext cctx = plan.cacheContext();
+        assert plan.isLocalSubquery();
 
-            QueryCursorImpl<List<?>> cur;
+        final GridCacheContext cctx = plan.cacheContext();
 
-            final ArrayList<List<?>> data = new ArrayList<>(plan.rowCount());
+        QueryCursorImpl<List<?>> cur;
 
-            QueryCursorImpl<List<?>> stepCur = new QueryCursorImpl<>(new Iterable<List<?>>() {
-                @Override public Iterator<List<?>> iterator() {
-                    try {
-                        Iterator<List<?>> it;
+        final ArrayList<List<?>> data = new ArrayList<>(plan.rowCount());
 
-                        if (!F.isEmpty(plan.selectQuery())) {
-                            GridQueryFieldsResult res = idx.queryLocalSqlFields(idx.schema(cctx.name()),
-                                plan.selectQuery(), F.asList(U.firstNotNull(args, X.EMPTY_OBJECT_ARRAY)),
-                                null, false, 0, null);
+        QueryCursorImpl<List<?>> stepCur = new QueryCursorImpl<>(new Iterable<List<?>>() {
+            @Override public Iterator<List<?>> iterator() {
+                try {
+                    Iterator<List<?>> it;
 
-                            it = res.iterator();
-                        }
-                        else
-                            it = plan.createRows(U.firstNotNull(args, X.EMPTY_OBJECT_ARRAY)).iterator();
+                    if (!F.isEmpty(plan.selectQuery())) {
+                        GridQueryFieldsResult res = idx.queryLocalSqlFields(idx.schema(cctx.name()),
+                            plan.selectQuery(), F.asList(U.firstNotNull(args, X.EMPTY_OBJECT_ARRAY)),
+                            null, false, 0, null);
 
-                        return new GridQueryCacheObjectsIterator(it, idx.objectContext(), cctx.keepBinary());
-                    }
-                    catch (IgniteCheckedException e) {
-                        throw new IgniteException(e);
+                        it = res.iterator();
                     }
-                }
-            }, null);
-
-            data.addAll(stepCur.getAll());
+                    else
+                        it = plan.createRows(U.firstNotNull(args, X.EMPTY_OBJECT_ARRAY)).iterator();
 
-            cur = new QueryCursorImpl<>(new Iterable<List<?>>() {
-                @Override public Iterator<List<?>> iterator() {
-                    return data.iterator();
+                    return new GridQueryCacheObjectsIterator(it, idx.objectContext(), cctx.keepBinary());
                 }
-            }, null);
-
-            if (plan.rowCount() == 1) {
-                IgniteBiTuple t = plan.processRow(cur.iterator().next());
+                catch (IgniteCheckedException e) {
+                    throw new IgniteException(e);
+                }
+            }
+        }, null);
 
-                streamer.addData(t.getKey(), t.getValue());
+        data.addAll(stepCur.getAll());
 
-                return 1;
+        cur = new QueryCursorImpl<>(new Iterable<List<?>>() {
+            @Override public Iterator<List<?>> iterator() {
+                return data.iterator();
             }
+        }, null);
 
-            Map<Object, Object> rows = new LinkedHashMap<>(plan.rowCount());
+        if (plan.rowCount() == 1) {
+            IgniteBiTuple t = plan.processRow(cur.iterator().next());
 
-            for (List<?> row : cur) {
-                final IgniteBiTuple t = plan.processRow(row);
+            streamer.addData(t.getKey(), t.getValue());
 
-                rows.put(t.getKey(), t.getValue());
-            }
+            return 1;
+        }
+
+        Map<Object, Object> rows = new LinkedHashMap<>(plan.rowCount());
 
-            streamer.addData(rows);
+        for (List<?> row : cur) {
+            final IgniteBiTuple t = plan.processRow(row);
 
-            return rows.size();
+            rows.put(t.getKey(), t.getValue());
         }
-        else
-            throw new IgniteSQLException("Only tuple based INSERT statements are supported in streaming mode",
-                IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+
+        streamer.addData(rows);
+
+        return rows.size();
     }
 
     /**
@@ -519,7 +513,7 @@ public class DmlStatementsProcessor {
                 .setPageSize(fieldsQry.getPageSize())
                 .setTimeout(fieldsQry.getTimeout(), TimeUnit.MILLISECONDS);
 
-            cur = (QueryCursorImpl<List<?>>)idx.querySqlFields(schemaName, newFieldsQry, true, true,
+            cur = (QueryCursorImpl<List<?>>)idx.querySqlFields(schemaName, newFieldsQry, null, true, true,
                 cancel).get(0);
         }
         else if (plan.hasRows())
@@ -610,7 +604,7 @@ public class DmlStatementsProcessor {
      * @return Update plan.
      */
     @SuppressWarnings({"unchecked", "ConstantConditions"})
-    private UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared p, SqlFieldsQuery fieldsQry,
+    UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared p, SqlFieldsQuery fieldsQry,
         boolean loc, @Nullable Integer errKeysPos) throws IgniteCheckedException {
         H2CachedStatementKey planKey = H2CachedStatementKey.forDmlStatement(schema, p.getSQL(), fieldsQry, loc);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java
index cfbb7bb..e9d9f90 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java
@@ -17,8 +17,16 @@
 
 package org.apache.ignite.internal.processors.query.h2;
 
+import java.lang.reflect.Constructor;
+import java.sql.Connection;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
 import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
@@ -33,12 +41,7 @@ import org.h2.table.IndexColumn;
 import org.h2.value.DataType;
 import org.h2.value.Value;
 
-import java.lang.reflect.Constructor;
-import java.sql.Connection;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
+import static org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.UPDATE_RESULT_META;
 
 /**
  * H2 utility methods.
@@ -267,4 +270,17 @@ public class H2Utils {
     private H2Utils() {
         // No-op.
     }
+
+    /**
+     * @return Single-column, single-row cursor with 0 as number of updated records.
+     */
+    @SuppressWarnings("unchecked")
+    public static QueryCursorImpl<List<?>> zeroCursor() {
+        QueryCursorImpl<List<?>> resCur = (QueryCursorImpl<List<?>>)new QueryCursorImpl(Collections.singletonList
+            (Collections.singletonList(0L)), null, false);
+
+        resCur.fieldsMeta(UPDATE_RESULT_META);
+
+        return resCur;
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 361623c..65f08b2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -92,6 +92,7 @@ import org.apache.ignite.internal.processors.query.IgniteSQLException;
 import org.apache.ignite.internal.processors.query.QueryField;
 import org.apache.ignite.internal.processors.query.QueryIndexDescriptorImpl;
 import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.processors.query.SqlClientContext;
 import org.apache.ignite.internal.processors.query.h2.database.H2RowFactory;
 import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex;
 import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasInnerIO;
@@ -100,6 +101,7 @@ import org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO;
 import org.apache.ignite.internal.processors.query.h2.database.io.H2LeafIO;
 import org.apache.ignite.internal.processors.query.h2.ddl.DdlStatementsProcessor;
 import org.apache.ignite.internal.processors.query.h2.dml.DmlUtils;
+import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlan;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2DefaultTableEngine;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2PlainRowFactory;
@@ -120,11 +122,12 @@ import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisito
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.sql.SqlParseException;
 import org.apache.ignite.internal.sql.SqlParser;
-import org.apache.ignite.internal.sql.command.SqlBulkLoadCommand;
 import org.apache.ignite.internal.sql.command.SqlAlterTableCommand;
+import org.apache.ignite.internal.sql.command.SqlBulkLoadCommand;
 import org.apache.ignite.internal.sql.command.SqlCommand;
 import org.apache.ignite.internal.sql.command.SqlCreateIndexCommand;
 import org.apache.ignite.internal.sql.command.SqlDropIndexCommand;
+import org.apache.ignite.internal.sql.command.SqlSetStreamingCommand;
 import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap;
 import org.apache.ignite.internal.util.GridEmptyCloseableIterator;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
@@ -149,7 +152,6 @@ import org.apache.ignite.spi.indexing.IndexingQueryFilterImpl;
 import org.h2.api.ErrorCode;
 import org.h2.api.JavaObjectSerializer;
 import org.h2.command.Prepared;
-import org.h2.command.dml.Insert;
 import org.h2.command.dml.NoOperation;
 import org.h2.engine.Session;
 import org.h2.engine.SysProperties;
@@ -191,7 +193,7 @@ import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType
 @SuppressWarnings({"UnnecessaryFullyQualifiedName", "NonFinalStaticVariableUsedInClassInitialization"})
 public class IgniteH2Indexing implements GridQueryIndexing {
     public static final Pattern INTERNAL_CMD_RE = Pattern.compile(
-        "^(create|drop)\\s+index|^alter\\s+table|^copy", Pattern.CASE_INSENSITIVE);
+        "^(create|drop)\\s+index|^alter\\s+table|^copy|^set", Pattern.CASE_INSENSITIVE);
 
     /*
      * Register IO for indexes.
@@ -500,10 +502,10 @@ public class IgniteH2Indexing implements GridQueryIndexing {
     }
 
     /** {@inheritDoc} */
-    @Override public PreparedStatement prepareNativeStatement(String schemaName, String sql) throws SQLException {
+    @Override public PreparedStatement prepareNativeStatement(String schemaName, String sql) {
         Connection conn = connectionForSchema(schemaName);
 
-        return prepareStatement(conn, sql, true);
+        return prepareStatementAndCaches(conn, sql);
     }
 
     /**
@@ -1013,7 +1015,55 @@ public class IgniteH2Indexing implements GridQueryIndexing {
             throw new IgniteSQLException(e);
         }
 
-        return dmlProc.streamUpdateQuery(streamer, stmt, params);
+        return dmlProc.streamUpdateQuery(schemaName, streamer, stmt, params);
+    }
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("ForLoopReplaceableByForEach")
+    @Override public List<Long> streamBatchedUpdateQuery(String schemaName, String qry, List<Object[]> params,
+        SqlClientContext cliCtx) throws IgniteCheckedException {
+        if (cliCtx == null || !cliCtx.isStream()) {
+            U.warn(log, "Connection is not in streaming mode.");
+
+            return zeroBatchedStreamedUpdateResult(params.size());
+        }
+
+        final Connection conn = connectionForSchema(schemaName);
+
+        final PreparedStatement stmt = prepareStatementAndCaches(conn, qry);
+
+        if (GridSqlQueryParser.checkMultipleStatements(stmt))
+            throw new IgniteSQLException("Multiple statements queries are not supported for streaming mode.",
+                IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+
+        checkStatementStreamable(stmt);
+
+        Prepared p = GridSqlQueryParser.prepared(stmt);
+
+        UpdatePlan plan = dmlProc.getPlanForStatement(schemaName, conn, p, null, true, null);
+
+        IgniteDataStreamer<?, ?> streamer = cliCtx.streamerForCache(plan.cacheContext().name());
+
+        assert streamer != null;
+
+        List<Long> res = new ArrayList<>(params.size());
+
+        for (int i = 0; i < params.size(); i++)
+            res.add(dmlProc.streamUpdateQuery(schemaName, streamer, stmt, params.get(i)));
+
+        return res;
+    }
+
+    /**
+     * @param size Result size.
+     * @return List of given size filled with 0Ls.
+     */
+    private static List<Long> zeroBatchedStreamedUpdateResult(int size) {
+        Long[] res = new Long[size];
+
+        Arrays.fill(res, 0);
+
+        return Arrays.asList(res);
     }
 
     /**
@@ -1399,7 +1449,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
             fqry.setTimeout(qry.getTimeout(), TimeUnit.MILLISECONDS);
 
         final QueryCursor<List<?>> res =
-            querySqlFields(schemaName, fqry, keepBinary, true, null).get(0);
+            querySqlFields(schemaName, fqry, null, keepBinary, true, null).get(0);
 
         final Iterable<Cache.Entry<K, V>> converted = new Iterable<Cache.Entry<K, V>>() {
             @Override public Iterator<Cache.Entry<K, V>> iterator() {
@@ -1435,19 +1485,21 @@ public class IgniteH2Indexing implements GridQueryIndexing {
      * Try executing query using native facilities.
      *
      * @param schemaName Schema name.
-     * @param qry Query.
+     * @param sql Query.
+     * @param cliCtx Client context, or {@code null} if not applicable.
      * @return Result or {@code null} if cannot parse/process this query.
      */
-    private List<FieldsQueryCursor<List<?>>> tryQueryDistributedSqlFieldsNative(String schemaName, SqlFieldsQuery qry) {
+    private List<FieldsQueryCursor<List<?>>> tryQueryDistributedSqlFieldsNative(String schemaName, String sql,
+        @Nullable SqlClientContext cliCtx) {
         // Heuristic check for fast return.
-        if (!INTERNAL_CMD_RE.matcher(qry.getSql().trim()).find())
+        if (!INTERNAL_CMD_RE.matcher(sql.trim()).find())
             return null;
 
         // Parse.
         SqlCommand cmd;
 
         try {
-            SqlParser parser = new SqlParser(schemaName, qry.getSql());
+            SqlParser parser = new SqlParser(schemaName, sql);
 
             cmd = parser.nextCommand();
 
@@ -1455,15 +1507,20 @@ public class IgniteH2Indexing implements GridQueryIndexing {
             if (parser.nextCommand() != null)
                 return null;
 
-            // Currently supported commands are: CREATE/DROP INDEX/COPY/ALTER TABLE
+            // Currently supported commands are:
+            // CREATE/DROP INDEX
+            // COPY
+            // ALTER TABLE
+            // SET STREAMING
             if (!(cmd instanceof SqlCreateIndexCommand || cmd instanceof SqlDropIndexCommand ||
-                cmd instanceof SqlBulkLoadCommand || cmd instanceof SqlAlterTableCommand))
+                cmd instanceof SqlAlterTableCommand || cmd instanceof SqlBulkLoadCommand ||
+                cmd instanceof SqlSetStreamingCommand))
                 return null;
         }
         catch (Exception e) {
             // Cannot parse, return.
             if (log.isDebugEnabled())
-                log.debug("Failed to parse SQL with native parser [qry=" + qry.getSql() + ", err=" + e + ']');
+                log.debug("Failed to parse SQL with native parser [qry=" + sql + ", err=" + e + ']');
 
             if (!IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_SQL_PARSER_DISABLE_H2_FALLBACK))
                 return null;
@@ -1473,24 +1530,40 @@ public class IgniteH2Indexing implements GridQueryIndexing {
             if (e instanceof SqlParseException)
                 code = ((SqlParseException)e).code();
 
-            throw new IgniteSQLException("Failed to parse DDL statement: " + qry.getSql() + ": " + e.getMessage(),
+            throw new IgniteSQLException("Failed to parse DDL statement: " + sql + ": " + e.getMessage(),
                 code, e);
         }
 
         // Execute.
         if (cmd instanceof SqlBulkLoadCommand) {
-            FieldsQueryCursor<List<?>> cursor = dmlProc.runNativeDmlStatement(qry.getSql(), cmd);
+            FieldsQueryCursor<List<?>> cursor = dmlProc.runNativeDmlStatement(sql, cmd);
 
             return Collections.singletonList(cursor);
         }
+        else if (cmd instanceof SqlSetStreamingCommand) {
+            if (cliCtx == null)
+                throw new IgniteSQLException("SET STREAMING command can only be executed from JDBC or ODBC driver.");
+
+            SqlSetStreamingCommand setCmd = (SqlSetStreamingCommand)cmd;
+
+            boolean on = setCmd.isTurnOn();
+
+            if (on)
+                cliCtx.enableStreaming(setCmd.allowOverwrite(), setCmd.flushFrequency(),
+                    setCmd.perNodeBufferSize(), setCmd.perNodeParallelOperations());
+            else
+                cliCtx.disableStreaming();
+
+            return Collections.singletonList(H2Utils.zeroCursor());
+        }
         else {
             try {
-                FieldsQueryCursor<List<?>> cursor = ddlProc.runDdlStatement(qry.getSql(), cmd);
+                FieldsQueryCursor<List<?>> cursor = ddlProc.runDdlStatement(sql, cmd);
 
                 return Collections.singletonList(cursor);
             }
             catch (IgniteCheckedException e) {
-                throw new IgniteSQLException("Failed to execute DDL statement [stmt=" + qry.getSql() + "]: "
+                throw new IgniteSQLException("Failed to execute DDL statement [stmt=" + sql + "]: "
                     + e.getMessage(), e);
             }
         }
@@ -1512,10 +1585,10 @@ public class IgniteH2Indexing implements GridQueryIndexing {
 
 
     /** {@inheritDoc} */
-    @SuppressWarnings("StringEquality")
+    @SuppressWarnings({"StringEquality", "unchecked"})
     @Override public List<FieldsQueryCursor<List<?>>> querySqlFields(String schemaName, SqlFieldsQuery qry,
-        boolean keepBinary, boolean failOnMultipleStmts, GridQueryCancel cancel) {
-        List<FieldsQueryCursor<List<?>>> res = tryQueryDistributedSqlFieldsNative(schemaName, qry);
+        @Nullable SqlClientContext cliCtx, boolean keepBinary, boolean failOnMultipleStmts, GridQueryCancel cancel) {
+        List<FieldsQueryCursor<List<?>>> res = tryQueryDistributedSqlFieldsNative(schemaName, qry.getSql(), cliCtx);
 
         if (res != null)
             return res;
@@ -1553,8 +1626,8 @@ public class IgniteH2Indexing implements GridQueryIndexing {
 
                 // We may use this cached statement only for local queries and non queries.
                 if (qry.isLocal() || !prepared.isQuery())
-                    return (List<FieldsQueryCursor<List<?>>>)doRunPrepared(schemaName, prepared, qry, null, null,
-                            keepBinary, cancel);
+                    return (List<FieldsQueryCursor<List<?>>>)doRunPrepared(schemaName, prepared, qry, null,
+                        null, keepBinary, cancel);
             }
         }
 
@@ -1602,12 +1675,11 @@ public class IgniteH2Indexing implements GridQueryIndexing {
      * @param twoStepQry Two-step query if this query must be executed in a distributed way.
      * @param meta Metadata for {@code twoStepQry}.
      * @param keepBinary Whether binary objects must not be deserialized automatically.
-     * @param cancel Query cancel state holder.
-     * @return Query result.
+     * @param cancel Query cancel state holder.    @return Query result.
      */
     private List<? extends FieldsQueryCursor<List<?>>> doRunPrepared(String schemaName, Prepared prepared,
-        SqlFieldsQuery qry, GridCacheTwoStepQuery twoStepQry, List<GridQueryFieldMetadata> meta, boolean keepBinary,
-        GridQueryCancel cancel) {
+        SqlFieldsQuery qry, GridCacheTwoStepQuery twoStepQry,
+        List<GridQueryFieldMetadata> meta, boolean keepBinary, GridQueryCancel cancel) {
         String sqlQry = qry.getSql();
 
         boolean loc = qry.isLocal();
@@ -2276,10 +2348,10 @@ public class IgniteH2Indexing implements GridQueryIndexing {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean isInsertStatement(PreparedStatement nativeStmt) {
-        Prepared prep = GridSqlQueryParser.prepared(nativeStmt);
-
-        return prep instanceof Insert;
+    @Override public void checkStatementStreamable(PreparedStatement nativeStmt) {
+        if (!GridSqlQueryParser.isStreamableInsertStatement(nativeStmt))
+            throw new IgniteSQLException("Streaming mode supports only INSERT commands without subqueries.",
+                IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
index 6f5b51f..d4f2b0e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
@@ -43,6 +43,7 @@ import org.apache.ignite.internal.processors.query.IgniteSQLException;
 import org.apache.ignite.internal.processors.query.QueryEntityEx;
 import org.apache.ignite.internal.processors.query.QueryField;
 import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.processors.query.h2.H2Utils;
 import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlterTableAddColumn;
@@ -106,7 +107,7 @@ public class DdlStatementsProcessor {
      * @throws IgniteCheckedException On error.
      */
     @SuppressWarnings("unchecked")
-    public FieldsQueryCursor<List<?>> runDdlStatement(String sql, SqlCommand cmd) throws IgniteCheckedException{
+    public FieldsQueryCursor<List<?>> runDdlStatement(String sql, SqlCommand cmd) throws IgniteCheckedException {
         IgniteInternalFuture fut;
 
         try {
@@ -211,12 +212,7 @@ public class DdlStatementsProcessor {
             if (fut != null)
                 fut.get();
 
-            QueryCursorImpl<List<?>> resCur = (QueryCursorImpl<List<?>>)new QueryCursorImpl(Collections.singletonList
-                (Collections.singletonList(0L)), null, false);
-
-            resCur.fieldsMeta(UPDATE_RESULT_META);
-
-            return resCur;
+            return H2Utils.zeroCursor();
         }
         catch (SchemaOperationException e) {
             throw convert(e);

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
index 10d485a..98fbb97 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
@@ -505,7 +505,7 @@ public final class UpdatePlan {
     /**
      * @return Local subquery flag.
      */
-    @Nullable public boolean isLocalSubquery() {
+    public boolean isLocalSubquery() {
         return isLocSubqry;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
index bced836..d897ac7 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
@@ -85,20 +85,21 @@ public final class UpdatePlanBuilder {
      * @param loc Local query flag.
      * @param idx Indexing.
      * @param conn Connection.
-     * @param fieldsQuery Original query.
+     * @param fieldsQry Original query.
      * @return Update plan.
      */
     public static UpdatePlan planForStatement(Prepared prepared, boolean loc, IgniteH2Indexing idx,
-        @Nullable Connection conn, @Nullable SqlFieldsQuery fieldsQuery, @Nullable Integer errKeysPos)
+        @Nullable Connection conn, @Nullable SqlFieldsQuery fieldsQry, @Nullable Integer errKeysPos)
         throws IgniteCheckedException {
-        assert !prepared.isQuery();
-
         GridSqlStatement stmt = new GridSqlQueryParser(false).parse(prepared);
 
         if (stmt instanceof GridSqlMerge || stmt instanceof GridSqlInsert)
-            return planForInsert(stmt, loc, idx, conn, fieldsQuery);
+            return planForInsert(stmt, loc, idx, conn, fieldsQry);
+        else if (stmt instanceof GridSqlUpdate || stmt instanceof GridSqlDelete)
+            return planForUpdate(stmt, loc, idx, conn, fieldsQry, errKeysPos);
         else
-            return planForUpdate(stmt, loc, idx, conn, fieldsQuery, errKeysPos);
+            throw new IgniteSQLException("Unsupported operation: " + prepared.getSQL(),
+                IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
index 04bc212..2d2c25c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
@@ -1993,6 +1993,18 @@ public class GridSqlQueryParser {
     }
 
     /**
+     * Check if passed statement is insert statement eligible for streaming.
+     *
+     * @param nativeStmt Native statement.
+     * @return {@code True} if streamable insert.
+     */
+    public static boolean isStreamableInsertStatement(PreparedStatement nativeStmt) {
+        Prepared prep = prepared(nativeStmt);
+
+        return prep instanceof Insert && INSERT_QUERY.get((Insert)prep) == null;
+    }
+
+    /**
      * @param cond Condition.
      * @param o Object.
      */

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
index 069bdd7..cf8bb2e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
@@ -140,7 +140,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
         SqlFieldsQuery qry = new SqlFieldsQuery("select f.productId, p.name, f.price " +
             "from FactPurchase f, \"replicated-prod\".DimProduct p where p.id = f.productId ");
 
-        for (List<?> o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
+        for (List<?> o : qryProc.querySqlFields(cache.context(), qry, null, false, true).get(0).getAll()) {
             X.println("___ -> " + o);
 
             set1.add((Integer)o.get(0));
@@ -154,7 +154,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
 
         qry = new SqlFieldsQuery("select productId from FactPurchase group by productId");
 
-        for (List<?> o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
+        for (List<?> o : qryProc.querySqlFields(cache.context(), qry, null, false, true).get(0).getAll()) {
             X.println("___ -> " + o);
 
             assertTrue(set0.add((Integer) o.get(0)));
@@ -173,7 +173,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
             "where p.id = f.productId " +
             "group by f.productId, p.name");
 
-        for (List<?> o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
+        for (List<?> o : qryProc.querySqlFields(cache.context(), qry, null, false, true).get(0).getAll()) {
             X.println("___ -> " + o);
 
             assertTrue(names.add((String)o.get(0)));
@@ -190,7 +190,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
             "group by f.productId, p.name " +
             "having s >= 15");
 
-        for (List<?> o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
+        for (List<?> o : qryProc.querySqlFields(cache.context(), qry, null, false, true).get(0).getAll()) {
             X.println("___ -> " + o);
 
             assertTrue(i(o, 1) >= 15);
@@ -203,7 +203,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
         qry = new SqlFieldsQuery("select top 3 distinct productId " +
             "from FactPurchase f order by productId desc ");
 
-        for (List<?> o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
+        for (List<?> o : qryProc.querySqlFields(cache.context(), qry, null, false, true).get(0).getAll()) {
             X.println("___ -> " + o);
 
             assertEquals(top--, o.get(0));
@@ -216,7 +216,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
         qry = new SqlFieldsQuery("select distinct productId " +
             "from FactPurchase f order by productId desc limit 2 offset 1");
 
-        for (List<?> o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
+        for (List<?> o : qryProc.querySqlFields(cache.context(), qry, null, false, true).get(0).getAll()) {
             X.println("___ -> " + o);
 
             assertEquals(top--, o.get(0));
@@ -256,13 +256,13 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
         GridTestUtils.assertThrows(log,
             new Callable<Object>() {
                 @Override public Object call() throws Exception {
-                    qryProc.querySqlFields(cache.context(), qry, false, true);
+                    qryProc.querySqlFields(cache.context(), qry, null, false, true);
 
                     return null;
                 }
             }, IgniteSQLException.class, "Multiple statements queries are not supported");
 
-        List<FieldsQueryCursor<List<?>>> cursors = qryProc.querySqlFields(cache.context(), qry, false, false);
+        List<FieldsQueryCursor<List<?>>> cursors = qryProc.querySqlFields(cache.context(), qry, null, false, false);
 
         assertEquals(2, cursors.size());
 
@@ -274,7 +274,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
         GridTestUtils.assertThrows(log,
             new Callable<Object>() {
                 @Override public Object call() throws Exception {
-                    qryProc.querySqlFields(cache.context(), qry, false, false);
+                    qryProc.querySqlFields(cache.context(), qry, null, false, false);
 
                     return null;
                 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index f004453..c01eaa6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -170,6 +170,7 @@ import org.apache.ignite.internal.processors.sql.SqlConnectorConfigurationValida
 import org.apache.ignite.internal.sql.SqlParserBulkLoadSelfTest;
 import org.apache.ignite.internal.sql.SqlParserCreateIndexSelfTest;
 import org.apache.ignite.internal.sql.SqlParserDropIndexSelfTest;
+import org.apache.ignite.internal.sql.SqlParserSetStreamingSelfTest;
 import org.apache.ignite.spi.communication.tcp.GridOrderedMessageCancelSelfTest;
 import org.apache.ignite.testframework.IgniteTestSuite;
 
@@ -187,6 +188,7 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
         suite.addTestSuite(SqlParserCreateIndexSelfTest.class);
         suite.addTestSuite(SqlParserDropIndexSelfTest.class);
         suite.addTestSuite(SqlParserBulkLoadSelfTest.class);
+        suite.addTestSuite(SqlParserSetStreamingSelfTest.class);
 
         suite.addTestSuite(SqlConnectorConfigurationValidationSelfTest.class);
         suite.addTestSuite(ClientConnectorConfigurationValidationSelfTest.class);


[50/50] [abbrv] ignite git commit: Merge remote-tracking branch 'remotes/origin/master' into ignite-zk

Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-zk

# Conflicts:
#	modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
#	modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
#	modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
#	modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
#	modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/00917728
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/00917728
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/00917728

Branch: refs/heads/ignite-zk
Commit: 00917728b527abe98a5b9e582d3bf749bb8e2beb
Parents: 23ac316
Author: sboikov <sb...@gridgain.com>
Authored: Mon Mar 19 09:26:05 2018 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Mar 19 09:26:05 2018 +0300

----------------------------------------------------------------------
 .../internal/managers/communication/GridIoMessageFactory.java   | 4 ++--
 .../ignite/internal/processors/cache/WalStateFinishMessage.java | 5 +++++
 .../internal/processors/cache/WalStateProposeMessage.java       | 5 +++++
 .../processors/cluster/ClusterMetricsUpdateMessage.java         | 2 +-
 .../continuous/ContinuousRoutineStartResultMessage.java         | 2 +-
 .../processors/cache/IgniteClusterActivateDeactivateTest.java   | 2 +-
 6 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/00917728/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 7ee55f9..d7fc31f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -893,12 +893,12 @@ public class GridIoMessageFactory implements MessageFactory {
 
                 break;
 
-            case 129:
+            case 130:
                 msg = new ClusterMetricsUpdateMessage();
 
                 break;
 
-            case 130:
+            case 131:
                 msg = new ContinuousRoutineStartResultMessage();
 
                 break;

http://git-wip-us.apache.org/repos/asf/ignite/blob/00917728/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
index 57f25d0..4afa403 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
@@ -67,6 +67,11 @@ public class WalStateFinishMessage extends WalStateAbstractMessage {
     }
 
     /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(WalStateFinishMessage.class, this, "super", super.toString());
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/00917728/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
index 747fd6a..b9d96fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
@@ -98,6 +98,11 @@ public class WalStateProposeMessage extends WalStateAbstractMessage {
     }
 
     /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(WalStateProposeMessage.class, this, "super", super.toString());
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/00917728/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
index 239dad5..90a5bd9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
@@ -138,7 +138,7 @@ public class ClusterMetricsUpdateMessage implements Message {
 
     /** {@inheritDoc} */
     @Override public short directType() {
-        return 129;
+        return 130;
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/00917728/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
index 9f40128..728380f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
@@ -186,7 +186,7 @@ public class ContinuousRoutineStartResultMessage implements Message {
 
     /** {@inheritDoc} */
     @Override public short directType() {
-        return 130;
+        return 131;
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/00917728/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 92577ad..838e56d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -1128,7 +1128,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
      * @throws Exception If failed.
      */
     private void stateChangeFailover3(boolean activate) throws Exception {
-        testDiscoSpi = true;
+        testReconnectSpi = true;
 
         startNodesAndBlockStatusChange(4, 0, 0, !activate);
 


[49/50] [abbrv] ignite git commit: Merge remote-tracking branch 'remotes/origin/master' into ignite-zk

Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-zk

# Conflicts:
#	modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
#	modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
#	modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
#	modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
#	modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/23ac3166
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/23ac3166
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/23ac3166

Branch: refs/heads/ignite-zk
Commit: 23ac31662375b2e314dc586830908331c58306ea
Parents: a65d955 98ba7b3
Author: sboikov <sb...@gridgain.com>
Authored: Mon Mar 19 09:21:34 2018 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Mar 19 09:21:34 2018 +0300

----------------------------------------------------------------------
 .gitignore                                      |    4 +
 NOTICE                                          |    2 +-
 RELEASE_NOTES.txt                               |   94 +
 assembly/NOTICE_FABRIC                          |    2 +-
 assembly/NOTICE_HADOOP                          |    2 +-
 bin/control.bat                                 |   16 +-
 bin/control.sh                                  |   15 +-
 bin/ignite.bat                                  |   24 +-
 bin/ignite.sh                                   |   15 +-
 bin/ignitevisorcmd.bat                          |   13 +-
 bin/ignitevisorcmd.sh                           |   12 +
 bin/include/build-classpath.bat                 |   13 +-
 bin/include/build-classpath.sh                  |    7 +-
 bin/include/functions.sh                        |   13 +-
 bin/include/parseargs.bat                       |    5 +-
 bin/include/setenv.bat                          |    8 +-
 bin/include/setenv.sh                           |    5 +
 config/ignite-log4j2.xml                        |    8 +-
 examples/README-LGPL.txt                        |    7 -
 examples/README.txt                             |   17 -
 examples/pom-standalone-lgpl.xml                |   66 +-
 examples/pom-standalone.xml                     |   66 +-
 examples/pom.xml                                |  100 +-
 .../examples/cluster/ClusterGroupExample.java   |   86 +
 .../ignite/examples/cluster/package-info.java   |   22 +
 .../computegrid/ComputeAsyncExample.java        |   17 +-
 .../computegrid/ComputeBroadcastExample.java    |   44 +-
 .../computegrid/ComputeCallableExample.java     |   18 +-
 .../computegrid/ComputeClosureExample.java      |   22 +-
 .../computegrid/ComputeRunnableExample.java     |   10 +-
 .../cluster/ClusterGroupExample.java            |   99 -
 .../computegrid/cluster/package-info.java       |   22 -
 .../examples/datagrid/CacheAffinityExample.java |   38 +-
 .../examples/datagrid/CacheApiExample.java      |   26 +-
 .../examples/datagrid/CacheAsyncApiExample.java |   37 +-
 ...heContinuousQueryWithTransformerExample.java |  150 +
 .../datagrid/CacheEntryProcessorExample.java    |   46 +-
 .../ignite/examples/datagrid/package-info.java  |    2 +-
 .../IgniteExecutorServiceExample.java           |    8 +-
 .../ignite/examples/events/EventsExample.java   |   32 +-
 .../examples/messaging/MessagingExample.java    |   95 +-
 .../messaging/MessagingPingPongExample.java     |   46 +-
 .../examples/ml/MLExamplesCommonArgs.java       |   31 +
 .../clustering/DatasetWithObviousStructure.java |  105 +
 .../ml/clustering/FuzzyCMeansExample.java       |  134 +
 .../ml/clustering/FuzzyCMeansLocalExample.java  |   95 +
 .../KMeansDistributedClustererExample.java      |   97 +
 .../clustering/KMeansLocalClustererExample.java |  106 +
 .../examples/ml/clustering/package-info.java    |   22 +
 .../AlgorithmSpecificDatasetExample.java        |  197 +
 .../ml/dataset/CacheBasedDatasetExample.java    |   90 +
 .../ml/dataset/LocalDatasetExample.java         |   84 +
 .../examples/ml/dataset/model/Person.java       |   58 +
 .../examples/ml/dataset/model/package-info.java |   22 +
 .../examples/ml/dataset/package-info.java       |   22 +
 .../ignite/examples/ml/genetic/change/Coin.java |   88 +
 .../OptimizeMakeChangeFitnessFunction.java      |  106 +
 .../change/OptimizeMakeChangeGAExample.java     |  207 +
 .../OptimizeMakeChangeTerminateCriteria.java    |   83 +
 .../helloworld/HelloWorldFitnessFunction.java   |   64 +
 .../genetic/helloworld/HelloWorldGAExample.java |  120 +
 .../helloworld/HelloWorldTerminateCriteria.java |   88 +
 .../ignite/examples/ml/genetic/movie/Movie.java |  133 +
 .../ml/genetic/movie/MovieFitnessFunction.java  |  105 +
 .../ml/genetic/movie/MovieGAExample.java        |  267 ++
 .../genetic/movie/MovieTerminateCriteria.java   |   88 +
 .../KNNClassificationExample.java               |  154 +
 .../ml/knn/classification/package-info.java     |   22 +
 .../ignite/examples/ml/knn/package-info.java    |   22 +
 .../ml/knn/regression/KNNRegressionExample.java |  155 +
 .../ml/knn/regression/package-info.java         |   22 +
 .../CholeskyDecompositionExample.java           |   80 +
 .../EigenDecompositionExample.java              |   69 +
 .../decompositions/LUDecompositionExample.java  |   83 +
 .../decompositions/QRDecompositionExample.java  |   82 +
 .../SingularValueDecompositionExample.java      |   70 +
 .../ml/math/decompositions/package-info.java    |   22 +
 .../ml/math/matrix/CacheMatrixExample.java      |  102 +
 .../ml/math/matrix/ExampleMatrixStorage.java    |  175 +
 .../math/matrix/MatrixCustomStorageExample.java |  141 +
 .../examples/ml/math/matrix/MatrixExample.java  |   79 +
 .../ml/math/matrix/MatrixExampleUtil.java       |   52 +
 .../ml/math/matrix/OffHeapMatrixExample.java    |   84 +
 .../matrix/SparseDistributedMatrixExample.java  |   70 +
 .../ml/math/matrix/SparseMatrixExample.java     |   84 +
 .../examples/ml/math/matrix/package-info.java   |   22 +
 .../ignite/examples/ml/math/package-info.java   |   22 +
 .../examples/ml/math/tracer/TracerExample.java  |   63 +
 .../examples/ml/math/tracer/package-info.java   |   22 +
 .../ml/math/vector/CacheVectorExample.java      |  109 +
 .../ml/math/vector/ExampleVectorStorage.java    |  127 +
 .../ml/math/vector/OffHeapVectorExample.java    |   78 +
 .../ml/math/vector/SparseVectorExample.java     |   76 +
 .../math/vector/VectorCustomStorageExample.java |  120 +
 .../examples/ml/math/vector/VectorExample.java  |   73 +
 .../examples/ml/math/vector/package-info.java   |   22 +
 .../examples/ml/nn/MLPGroupTrainerExample.java  |  140 +
 .../examples/ml/nn/MLPLocalTrainerExample.java  |  161 +
 .../ignite/examples/ml/nn/package-info.java     |   22 +
 .../apache/ignite/examples/ml/package-info.java |   22 +
 .../ml/preprocessing/NormalizationExample.java  |  109 +
 .../examples/ml/preprocessing/package-info.java |   22 +
 ...nWithLSQRTrainerAndNormalizationExample.java |  184 +
 ...dLinearRegressionWithLSQRTrainerExample.java |  170 +
 ...tedLinearRegressionWithQRTrainerExample.java |  136 +
 ...edLinearRegressionWithSGDTrainerExample.java |  137 +
 .../ml/regression/linear/package-info.java      |   22 +
 .../examples/ml/regression/package-info.java    |   22 +
 .../binary/SVMBinaryClassificationExample.java  |  233 ++
 .../examples/ml/svm/binary/package-info.java    |   22 +
 .../ignite/examples/ml/svm/package-info.java    |   22 +
 .../examples/ml/trees/DecisionTreesExample.java |  354 ++
 .../ignite/examples/ml/trees/package-info.java  |   22 +
 .../ignite/examples/sql/SqlDmlExample.java      |   12 +-
 .../ignite/examples/sql/SqlJdbcCopyExample.java |  107 +
 .../streaming/StreamTransformerExample.java     |   28 +-
 .../streaming/StreamVisitorExample.java         |   84 +-
 .../ignite/examples/streaming/package-info.java |    1 +
 .../java8/cluster/ClusterGroupExample.java      |   86 -
 .../examples/java8/cluster/package-info.java    |   22 -
 .../java8/computegrid/ComputeAsyncExample.java  |   75 -
 .../computegrid/ComputeBroadcastExample.java    |  102 -
 .../computegrid/ComputeCallableExample.java     |   75 -
 .../computegrid/ComputeClosureExample.java      |   71 -
 .../computegrid/ComputeRunnableExample.java     |   64 -
 .../java8/computegrid/package-info.java         |   22 -
 .../java8/datagrid/CacheAffinityExample.java    |  142 -
 .../java8/datagrid/CacheApiExample.java         |  110 -
 .../java8/datagrid/CacheAsyncApiExample.java    |   90 -
 .../datagrid/CacheEntryProcessorExample.java    |  152 -
 .../examples/java8/datagrid/package-info.java   |   22 -
 .../IgniteExecutorServiceExample.java           |   70 -
 .../java8/datastructures/package-info.java      |   22 -
 .../examples/java8/events/EventsExample.java    |  135 -
 .../examples/java8/events/package-info.java     |   22 -
 .../java8/messaging/MessagingExample.java       |  161 -
 .../messaging/MessagingPingPongExample.java     |  113 -
 .../examples/java8/messaging/package-info.java  |   22 -
 .../ignite/examples/java8/package-info.java     |   23 -
 .../streaming/StreamTransformerExample.java     |  108 -
 .../java8/streaming/StreamVisitorExample.java   |  177 -
 .../examples/java8/streaming/package-info.java  |   22 -
 .../clustering/DatasetWithObviousStructure.java |  105 -
 .../ml/clustering/FuzzyCMeansExample.java       |  134 -
 .../ml/clustering/FuzzyCMeansLocalExample.java  |   95 -
 .../KMeansDistributedClustererExample.java      |   97 -
 .../clustering/KMeansLocalClustererExample.java |  106 -
 .../examples/ml/clustering/package-info.java    |   22 -
 .../KNNClassificationExample.java               |  155 -
 .../ml/knn/classification/package-info.java     |   22 -
 .../ignite/examples/ml/knn/datasets/README.md   |    2 -
 .../ml/knn/datasets/cleared_machines.txt        |  209 -
 .../ignite/examples/ml/knn/datasets/iris.txt    |  150 -
 .../ignite/examples/ml/knn/package-info.java    |   22 -
 .../ml/knn/regression/KNNRegressionExample.java |  157 -
 .../ml/knn/regression/package-info.java         |   22 -
 .../CholeskyDecompositionExample.java           |   80 -
 .../EigenDecompositionExample.java              |   69 -
 .../decompositions/LUDecompositionExample.java  |   83 -
 .../decompositions/QRDecompositionExample.java  |   82 -
 .../SingularValueDecompositionExample.java      |   70 -
 .../ml/math/decompositions/package-info.java    |   22 -
 .../ml/math/matrix/CacheMatrixExample.java      |  102 -
 .../ml/math/matrix/ExampleMatrixStorage.java    |  175 -
 .../math/matrix/MatrixCustomStorageExample.java |  141 -
 .../examples/ml/math/matrix/MatrixExample.java  |   79 -
 .../ml/math/matrix/MatrixExampleUtil.java       |   52 -
 .../ml/math/matrix/OffHeapMatrixExample.java    |   84 -
 .../matrix/SparseDistributedMatrixExample.java  |   70 -
 .../ml/math/matrix/SparseMatrixExample.java     |   84 -
 .../examples/ml/math/matrix/package-info.java   |   22 -
 .../ignite/examples/ml/math/package-info.java   |   22 -
 .../examples/ml/math/tracer/TracerExample.java  |   63 -
 .../examples/ml/math/tracer/package-info.java   |   22 -
 .../ml/math/vector/CacheVectorExample.java      |  109 -
 .../ml/math/vector/ExampleVectorStorage.java    |  127 -
 .../ml/math/vector/OffHeapVectorExample.java    |   78 -
 .../ml/math/vector/SparseVectorExample.java     |   76 -
 .../math/vector/VectorCustomStorageExample.java |  120 -
 .../examples/ml/math/vector/VectorExample.java  |   73 -
 .../examples/ml/math/vector/package-info.java   |   22 -
 .../examples/ml/nn/MLPGroupTrainerExample.java  |  141 -
 .../examples/ml/nn/MLPLocalTrainerExample.java  |  161 -
 .../ignite/examples/ml/nn/package-info.java     |   22 -
 .../apache/ignite/examples/ml/package-info.java |   22 -
 ...tedLinearRegressionExampleWithQRTrainer.java |  136 -
 ...edLinearRegressionExampleWithSGDTrainer.java |  137 -
 .../ml/regression/linear/package-info.java      |   22 -
 .../examples/ml/regression/package-info.java    |   22 -
 .../ignite/examples/ml/trees/MNISTExample.java  |  261 --
 .../ignite/examples/ml/trees/package-info.java  |   22 -
 examples/src/main/resources/cityBulkLoad.csv    |    3 +
 examples/src/main/resources/datasets/README.md  |    2 +
 .../resources/datasets/cleared_machines.txt     |  209 +
 examples/src/main/resources/datasets/iris.txt   |  150 +
 examples/src/main/resources/person.json         |   10 +
 examples/src/main/resources/personBulkLoad.csv  |    4 +
 .../examples/ScalarSnowflakeSchemaExample.scala |    4 +-
 .../spark/IgniteDataFrameWriteExample.scala     |  179 +
 .../ignite/examples/BasicExamplesSelfTest.java  |   46 +-
 .../CacheContinuousQueryExamplesSelfTest.java   |   48 +
 .../CacheExamplesMultiNodeSelfTest.java         |   16 -
 .../ignite/examples/CacheExamplesSelfTest.java  |  254 +-
 .../examples/CheckpointExamplesSelfTest.java    |   29 +-
 .../examples/ClusterGroupExampleSelfTest.java   |   17 +-
 .../examples/ContinuationExamplesSelfTest.java  |   14 +-
 .../ContinuousMapperExamplesSelfTest.java       |   12 +-
 .../DeploymentExamplesMultiNodeSelfTest.java    |   12 +-
 .../examples/DeploymentExamplesSelfTest.java    |   13 +-
 .../ignite/examples/EventsExamplesSelfTest.java |    2 +-
 .../ignite/examples/IgfsExamplesSelfTest.java   |   38 +-
 .../examples/LifecycleExamplesSelfTest.java     |   14 +-
 .../MemcacheRestExamplesMultiNodeSelfTest.java  |   12 +-
 .../examples/MemcacheRestExamplesSelfTest.java  |   36 +-
 .../examples/MessagingExamplesSelfTest.java     |   15 +-
 .../examples/MonteCarloExamplesSelfTest.java    |   14 +-
 .../examples/SpringBeanExamplesSelfTest.java    |   14 +-
 .../ignite/examples/TaskExamplesSelfTest.java   |   34 +-
 .../IgniteExamplesJ8SelfTestSuite.java          |   80 +
 .../testsuites/IgniteExamplesMLTestSuite.java   |  151 +
 .../testsuites/IgniteExamplesSelfTestSuite.java |    7 +-
 .../BasicExamplesMultiNodeSelfTest.java         |   28 -
 .../java8/examples/BasicExamplesSelfTest.java   |   89 -
 .../CacheExamplesMultiNodeSelfTest.java         |   38 -
 .../java8/examples/CacheExamplesSelfTest.java   |  155 -
 .../examples/CheckpointExamplesSelfTest.java    |   47 -
 .../examples/ClusterGroupExampleSelfTest.java   |   40 -
 .../ContinuationExamplesMultiNodeSelfTest.java  |   28 -
 .../examples/ContinuationExamplesSelfTest.java  |   37 -
 ...ntinuousMapperExamplesMultiNodeSelfTest.java |   28 -
 .../ContinuousMapperExamplesSelfTest.java       |   35 -
 .../DeploymentExamplesMultiNodeSelfTest.java    |   32 -
 .../examples/DeploymentExamplesSelfTest.java    |   36 -
 .../EventsExamplesMultiNodeSelfTest.java        |   28 -
 .../java8/examples/EventsExamplesSelfTest.java  |   33 -
 .../java8/examples/IgfsExamplesSelfTest.java    |   55 -
 .../examples/LifecycleExamplesSelfTest.java     |   37 -
 .../MemcacheRestExamplesMultiNodeSelfTest.java  |   33 -
 .../examples/MemcacheRestExamplesSelfTest.java  |   44 -
 .../examples/MessagingExamplesSelfTest.java     |   55 -
 .../MonteCarloExamplesMultiNodeSelfTest.java    |   28 -
 .../examples/MonteCarloExamplesSelfTest.java    |   78 -
 .../examples/SpringBeanExamplesSelfTest.java    |   37 -
 .../examples/TaskExamplesMultiNodeSelfTest.java |   28 -
 .../java8/examples/TaskExamplesSelfTest.java    |   43 -
 .../IgniteExamplesJ8SelfTestSuite.java          |   80 -
 .../testsuites/IgniteExamplesMLTestSuite.java   |  140 -
 .../spark/examples/IgniteDataFrameSelfTest.java |    9 +
 modules/aop/pom.xml                             |   13 +-
 .../apache/ignite/gridify/BasicAopSelfTest.java |   26 +-
 .../testsuites/IgniteAopSelfTestSuite.java      |   12 +-
 modules/apache-license-gen/pom.xml              |    2 +-
 modules/aws/pom.xml                             |    2 +-
 modules/benchmarks/pom.xml                      |   17 +-
 .../jmh/notify/JmhWaitStategyBenchmark.java     |    7 +-
 modules/camel/pom.xml                           |    2 +-
 modules/cassandra/pom.xml                       |    8 +-
 modules/cassandra/serializers/pom.xml           |    9 +-
 .../cassandra/serializer/package-info.java      |   21 +
 modules/cassandra/store/pom.xml                 |   14 +-
 .../store/cassandra/CassandraCacheStore.java    |    3 +-
 .../cassandra/common/PropertyMappingHelper.java |    7 +-
 .../cassandra/session/CassandraSessionImpl.java |    9 +-
 .../ignite/tests/CassandraSessionImplTest.java  |  194 +
 .../ignite/tests/load/PersonGenerator.java      |    2 +-
 .../org/apache/ignite/tests/pojos/Person.java   |   12 +-
 .../apache/ignite/tests/pojos/SimplePerson.java |    8 +-
 .../apache/ignite/tests/utils/TestsHelper.java  |    4 +-
 modules/clients/pom.xml                         |    2 +-
 .../client/impl/ClientCacheFlagsCodecTest.java  |   35 +-
 .../jdbc2/JdbcAbstractDmlStatementSelfTest.java |    2 +-
 .../internal/jdbc2/JdbcBulkLoadSelfTest.java    |  185 +
 .../internal/jdbc2/JdbcErrorsSelfTest.java      |   12 +-
 .../jdbc2/JdbcNoCacheStreamingSelfTest.java     |  182 +
 .../internal/jdbc2/JdbcStreamingSelfTest.java   |  220 +-
 .../jdbc2/JdbcStreamingToPublicCacheTest.java   |  148 +
 .../JettyRestProcessorAbstractSelfTest.java     | 1182 ++++--
 .../rest/JettyRestProcessorSignedSelfTest.java  |    5 +-
 .../internal/processors/rest/SimplePerson.java  |    4 +-
 .../ignite/jdbc/JdbcErrorsAbstractSelfTest.java |  257 +-
 .../jdbc/suite/IgniteJdbcDriverTestSuite.java   |   34 +-
 .../jdbc/thin/JdbcThinAbstractSelfTest.java     |   79 +
 .../ignite/jdbc/thin/JdbcThinBatchSelfTest.java |  509 ++-
 .../thin/JdbcThinBulkLoadAbstractSelfTest.java  |  601 +++
 ...inBulkLoadAtomicPartitionedNearSelfTest.java |   39 +
 ...bcThinBulkLoadAtomicPartitionedSelfTest.java |   39 +
 ...dbcThinBulkLoadAtomicReplicatedSelfTest.java |   39 +
 ...oadTransactionalPartitionedNearSelfTest.java |   39 +
 ...ulkLoadTransactionalPartitionedSelfTest.java |   39 +
 ...BulkLoadTransactionalReplicatedSelfTest.java |   39 +
 .../thin/JdbcThinComplexDmlDdlSelfTest.java     |    2 +-
 .../jdbc/thin/JdbcThinConnectionSSLTest.java    |  479 +++
 .../jdbc/thin/JdbcThinConnectionSelfTest.java   |   46 +-
 .../JdbcThinDynamicIndexAbstractSelfTest.java   |    1 -
 .../jdbc/thin/JdbcThinErrorsSelfTest.java       |   16 +-
 .../jdbc/thin/JdbcThinLocalQueriesSelfTest.java |   73 +
 .../jdbc/thin/JdbcThinNoDefaultSchemaTest.java  |    2 +-
 .../jdbc/thin/JdbcThinStreamingSelfTest.java    |  486 +++
 .../ignite/jdbc/thin/JdbcThinTcpIoTest.java     |  165 +
 .../thin/JdbcThinWalModeChangeSelfTest.java     |  138 +
 .../clients/src/test/resources/bulkload0.csv    |    0
 .../clients/src/test/resources/bulkload1.csv    |    1 +
 .../clients/src/test/resources/bulkload2.csv    |    2 +
 .../src/test/resources/bulkload2_utf.csv        |    2 +
 modules/cloud/pom.xml                           |    2 +-
 modules/codegen/pom.xml                         |   11 +-
 modules/compatibility/pom.xml                   |    2 +-
 .../DummyPersistenceCompatibilityTest.java      |   27 +-
 .../FoldersReuseCompatibilityTest.java          |    2 +-
 .../IgniteUuidCompatibilityTest.java            |  189 +
 ...tingToWalV2SerializerWithCompactionTest.java |    2 +-
 .../junits/IgniteCompatibilityNodeRunner.java   |    1 +
 .../testframework/util/MavenUtils.java          |   28 +-
 .../IgniteCompatibilityBasicTestSuite.java      |    3 +
 modules/core/pom.xml                            |   13 +-
 modules/core/src/main/java/META-INF/NOTICE      |    2 +-
 .../org/apache/ignite/DataRegionMetrics.java    |   38 +
 .../src/main/java/org/apache/ignite/Ignite.java |    4 +
 .../java/org/apache/ignite/IgniteCluster.java   |   86 +
 .../java/org/apache/ignite/IgniteLogger.java    |   92 +-
 .../apache/ignite/IgniteSystemProperties.java   |  138 +-
 .../org/apache/ignite/cache/CacheMetrics.java   |    5 +
 .../ignite/cache/PartitionLossPolicy.java       |    4 +
 .../cache/eviction/AbstractEvictionPolicy.java  |    4 +-
 .../igfs/IgfsPerBlockLruEvictionPolicy.java     |    4 +-
 .../eviction/sorted/SortedEvictionPolicy.java   |    4 +-
 .../cache/query/AbstractContinuousQuery.java    |  202 +
 .../cache/query/BulkLoadContextCursor.java      |   97 +
 .../ignite/cache/query/ContinuousQuery.java     |  172 +-
 .../query/ContinuousQueryWithTransformer.java   |  192 +
 .../cache/store/jdbc/CacheJdbcBlobStore.java    |    6 +-
 .../org/apache/ignite/cluster/BaselineNode.java |   73 +
 .../org/apache/ignite/cluster/ClusterNode.java  |    8 +-
 .../configuration/AtomicConfiguration.java      |    2 +-
 .../configuration/CacheConfiguration.java       |   29 +-
 .../ClientConnectorConfiguration.java           |  200 +
 .../configuration/DataRegionConfiguration.java  |    8 +-
 .../configuration/DataStorageConfiguration.java |   35 +-
 .../configuration/IgniteConfiguration.java      |   37 +
 .../configuration/IgniteReflectionFactory.java  |   43 +-
 .../PersistentStoreConfiguration.java           |    7 +-
 .../apache/ignite/configuration/WALMode.java    |   15 +-
 .../org/apache/ignite/internal/ClassSet.java    |  108 +
 .../apache/ignite/internal/GridComponent.java   |    3 +
 .../ignite/internal/GridKernalContext.java      |   22 +-
 .../ignite/internal/GridKernalContextImpl.java  |   61 +-
 .../apache/ignite/internal/GridLoggerProxy.java |   25 +
 .../internal/GridNodeOrderComparator.java       |   45 -
 .../ignite/internal/GridPluginComponent.java    |    5 +
 .../org/apache/ignite/internal/GridTopic.java   |    3 +
 .../apache/ignite/internal/IgniteKernal.java    |  205 +-
 .../ignite/internal/IgniteVersionUtils.java     |    2 +-
 .../org/apache/ignite/internal/IgnitionEx.java  |   39 +-
 .../ignite/internal/LongJVMPauseDetector.java   |  167 +
 .../ignite/internal/MarshallerContextImpl.java  |   41 +-
 .../apache/ignite/internal/NodeInvalidator.java |   53 +
 .../internal/binary/BinaryClassDescriptor.java  |  384 +-
 .../ignite/internal/binary/BinaryContext.java   |   13 +-
 .../ignite/internal/binary/BinaryEnumCache.java |    4 +-
 .../internal/binary/BinaryFieldAccessor.java    |   61 +-
 .../internal/binary/BinaryObjectExImpl.java     |   69 +-
 .../ignite/internal/binary/BinaryUtils.java     |    6 -
 .../internal/client/GridClientCacheFlag.java    |   48 +
 .../internal/client/impl/GridClientImpl.java    |   28 +-
 .../impl/connection/GridClientConnection.java   |   18 +-
 .../GridClientConnectionManagerAdapter.java     |   16 +-
 .../connection/GridClientNioTcpConnection.java  |   14 +-
 .../GridClientOptimizedMarshaller.java          |    4 +-
 .../router/impl/GridRouterClientImpl.java       |    4 +-
 .../client/router/impl/GridTcpRouterImpl.java   |   25 +-
 .../impl/GridTcpRouterNioListenerAdapter.java   |    5 +
 .../cluster/ClusterNodeLocalMapImpl.java        |    4 +-
 .../internal/cluster/DetachedClusterNode.java   |  121 +
 .../cluster/IgniteClusterAsyncImpl.java         |   40 +
 .../internal/cluster/IgniteClusterImpl.java     |  248 ++
 .../internal/cluster/NodeOrderComparator.java   |   61 +
 .../cluster/NodeOrderLegacyComparator.java      |   46 +
 .../ignite/internal/commandline/Arguments.java  |  117 +
 .../internal/commandline/CommandHandler.java    |  690 +++-
 .../internal/direct/DirectMessageWriter.java    |    9 +
 .../direct/stream/DirectByteBufferStream.java   |    6 +
 .../stream/v1/DirectByteBufferStreamImplV1.java |    8 +
 .../stream/v2/DirectByteBufferStreamImplV2.java |   11 +
 .../internal/events/DiscoveryCustomEvent.java   |   34 +
 .../ignite/internal/igfs/common/IgfsLogger.java |   12 +-
 .../ignite/internal/jdbc/JdbcResultSet.java     |    2 +-
 .../jdbc/thin/ConnectionProperties.java         |  206 +
 .../jdbc/thin/ConnectionPropertiesImpl.java     |  239 +-
 .../internal/jdbc/thin/JdbcThinConnection.java  |  117 +
 .../jdbc/thin/JdbcThinPreparedStatement.java    |   22 +-
 .../internal/jdbc/thin/JdbcThinSSLUtil.java     |  332 ++
 .../internal/jdbc/thin/JdbcThinStatement.java   |  219 +-
 .../internal/jdbc/thin/JdbcThinTcpIo.java       |  139 +-
 .../ignite/internal/jdbc2/JdbcConnection.java   |   12 +-
 .../jdbc2/JdbcQueryMultipleStatementsTask.java  |    2 +-
 .../ignite/internal/jdbc2/JdbcQueryTask.java    |   12 +-
 .../ignite/internal/jdbc2/JdbcResultSet.java    |    2 +-
 .../jdbc2/JdbcStreamedPreparedStatement.java    |    2 +-
 .../internal/managers/GridManagerAdapter.java   |   19 +
 .../checkpoint/GridCheckpointManager.java       |    4 +-
 .../managers/communication/GridIoManager.java   |   29 +-
 .../communication/GridIoMessageFactory.java     |   18 +-
 .../managers/deployment/GridDeployment.java     |   16 +-
 .../deployment/GridDeploymentClassLoader.java   |    6 +-
 .../deployment/GridDeploymentLocalStore.java    |   29 +-
 .../GridDeploymentPerVersionStore.java          |    6 +-
 .../managers/discovery/ClusterMetricsImpl.java  |  369 ++
 .../managers/discovery/ConsistentIdMapper.java  |   96 +-
 .../internal/managers/discovery/DiscoCache.java |  138 +-
 .../discovery/GridDiscoveryManager.java         |  264 +-
 .../eventstorage/GridEventStorageManager.java   |    4 +-
 .../optimized/OptimizedMarshaller.java          |    4 +-
 .../optimized/OptimizedMarshallerUtils.java     |    4 -
 .../optimized/OptimizedObjectInputStream.java   |  108 +-
 .../optimized/OptimizedObjectOutputStream.java  |   97 +-
 .../mem/unsafe/UnsafeMemoryProvider.java        |   12 +-
 .../internal/pagemem/PageIdAllocator.java       |    5 +-
 .../ignite/internal/pagemem/PageMemory.java     |    5 +
 .../pagemem/impl/PageMemoryNoStoreImpl.java     |    7 +-
 .../pagemem/store/IgnitePageStoreManager.java   |   20 +-
 .../pagemem/wal/IgniteWriteAheadLogManager.java |   17 +-
 .../internal/pagemem/wal/StorageException.java  |   16 +-
 .../wal/record/BaselineTopologyRecord.java      |   76 +
 .../internal/pagemem/wal/record/CacheState.java |   24 +
 .../internal/pagemem/wal/record/DataEntry.java  |    4 +-
 .../internal/pagemem/wal/record/DataRecord.java |    6 +-
 .../pagemem/wal/record/ExchangeRecord.java      |   86 +
 .../pagemem/wal/record/FilteredRecord.java      |    3 +
 .../pagemem/wal/record/MetastoreDataRecord.java |   62 +
 .../pagemem/wal/record/PageSnapshot.java        |   12 +-
 .../pagemem/wal/record/SnapshotRecord.java      |   12 +
 .../pagemem/wal/record/TimeStampRecord.java     |    8 +
 .../internal/pagemem/wal/record/TxRecord.java   |   50 +-
 .../internal/pagemem/wal/record/WALRecord.java  |   37 +-
 .../wal/record/WalRecordCacheGroupAware.java    |   28 +
 .../delta/DataPageInsertFragmentRecord.java     |    5 +-
 .../wal/record/delta/DataPageInsertRecord.java  |    5 +-
 .../wal/record/delta/DataPageRemoveRecord.java  |    5 +-
 .../delta/DataPageSetFreeListPageRecord.java    |    5 +-
 .../wal/record/delta/DataPageUpdateRecord.java  |    5 +-
 .../wal/record/delta/PageDeltaRecord.java       |    9 +-
 .../record/delta/PartitionDestroyRecord.java    |    9 +-
 .../record/delta/PartitionMetaStateRecord.java  |    9 +-
 .../processors/GridProcessorAdapter.java        |    5 +
 .../affinity/GridAffinityAssignmentCache.java   |  108 +-
 .../affinity/GridAffinityProcessor.java         |    4 +-
 .../bulkload/BulkLoadAckClientParameters.java   |   92 +
 .../bulkload/BulkLoadCacheWriter.java           |   31 +
 .../processors/bulkload/BulkLoadCsvFormat.java  |  159 +
 .../processors/bulkload/BulkLoadCsvParser.java  |   65 +
 .../processors/bulkload/BulkLoadFormat.java     |   33 +
 .../processors/bulkload/BulkLoadParser.java     |   61 +
 .../processors/bulkload/BulkLoadProcessor.java  |  104 +
 .../bulkload/BulkLoadStreamerWriter.java        |   65 +
 .../bulkload/pipeline/CharsetDecoderBlock.java  |  132 +
 .../pipeline/CsvLineProcessorBlock.java         |   70 +
 .../bulkload/pipeline/LineSplitterBlock.java    |   72 +
 .../bulkload/pipeline/PipelineBlock.java        |   66 +
 .../bulkload/pipeline/StrListAppenderBlock.java |   52 +
 .../cache/CacheAffinitySharedManager.java       |  188 +-
 .../cache/CacheClusterMetricsMXBeanImpl.java    |    5 +
 .../cache/CacheConfigurationOverride.java       |  165 +
 .../processors/cache/CacheGroupContext.java     |   97 +-
 .../processors/cache/CacheGroupData.java        |   36 +-
 .../processors/cache/CacheGroupDescriptor.java  |   75 +-
 .../cache/CacheJoinNodeDiscoveryData.java       |    2 +-
 .../cache/CacheLocalMetricsMXBeanImpl.java      |    5 +
 .../processors/cache/CacheMetricsImpl.java      |  332 +-
 .../processors/cache/CacheMetricsSnapshot.java  |   35 +-
 .../cache/CacheStoreBalancingWrapper.java       |    4 +-
 .../cache/CacheWeakQueryIteratorsHolder.java    |    4 +-
 .../processors/cache/ClusterCachesInfo.java     |   77 +-
 .../cache/DynamicCacheDescriptor.java           |   19 +-
 .../processors/cache/ExchangeActions.java       |   33 +-
 .../cache/ExchangeDiscoveryEvents.java          |   29 +-
 .../cache/GatewayProtectedCacheProxy.java       |   14 +
 .../processors/cache/GridCacheAdapter.java      |   26 +-
 .../cache/GridCacheConcurrentMapImpl.java       |   14 +-
 .../processors/cache/GridCacheContext.java      |   72 +-
 .../cache/GridCacheDeploymentManager.java       |    6 +-
 .../processors/cache/GridCacheEntryEx.java      |   24 +
 .../processors/cache/GridCacheEventManager.java |    3 +-
 .../processors/cache/GridCacheIoManager.java    |   12 +-
 .../cache/GridCacheLocalConcurrentMap.java      |    4 +-
 .../processors/cache/GridCacheMapEntry.java     | 1034 +++--
 .../processors/cache/GridCacheMvccManager.java  |   13 +-
 .../GridCachePartitionExchangeManager.java      |   93 +-
 .../processors/cache/GridCachePreloader.java    |   16 +-
 .../cache/GridCachePreloaderAdapter.java        |   12 +-
 .../processors/cache/GridCacheProcessor.java    |  412 +-
 .../cache/GridCacheSharedContext.java           |   58 +-
 .../cache/GridCacheSharedManager.java           |    5 +
 .../cache/GridCacheSharedManagerAdapter.java    |    5 +
 .../processors/cache/GridCacheTtlManager.java   |    8 +-
 .../cache/GridCacheUpdateTxResult.java          |   21 +-
 .../processors/cache/GridCacheUtils.java        |   24 +-
 .../cache/GridDeferredAckMessageSender.java     |   12 +-
 .../cache/IgniteCacheOffheapManager.java        |   14 +-
 .../cache/IgniteCacheOffheapManagerImpl.java    |  126 +-
 .../processors/cache/IgniteCacheProxy.java      |   41 +-
 .../processors/cache/IgniteCacheProxyImpl.java  |  172 +-
 .../cache/IgniteRebalanceIterator.java          |   23 +-
 .../cache/LocalJoinCachesContext.java           |  127 +
 .../processors/cache/StateChangeRequest.java    |   42 +
 .../processors/cache/StoredCacheData.java       |    9 +-
 .../cache/WalStateAbstractMessage.java          |  135 +
 .../processors/cache/WalStateAckMessage.java    |  221 +
 .../cache/WalStateDistributedProcess.java       |  172 +
 .../processors/cache/WalStateFinishMessage.java |   73 +
 .../processors/cache/WalStateManager.java       |  914 +++++
 .../cache/WalStateNodeLeaveExchangeTask.java    |   57 +
 .../cache/WalStateProposeMessage.java           |  104 +
 .../processors/cache/WalStateResult.java        |   94 +
 .../cache/binary/BinaryMetadataFileStore.java   |    4 +-
 .../cache/binary/BinaryMetadataTransport.java   |   14 +-
 .../binary/CacheObjectBinaryProcessorImpl.java  |  123 +-
 .../CacheDataStructuresManager.java             |    8 +-
 .../distributed/GridCacheTxFinishSync.java      |    6 +-
 .../distributed/GridDistributedCacheEntry.java  |   84 +-
 .../GridDistributedTxRemoteAdapter.java         |   19 +-
 .../dht/GridClientPartitionTopology.java        |   34 +-
 .../dht/GridDhtAssignmentFetchFuture.java       |    4 +-
 .../distributed/dht/GridDhtCacheAdapter.java    |    8 +-
 .../distributed/dht/GridDhtCacheEntry.java      |  295 +-
 .../distributed/dht/GridDhtLocalPartition.java  |  488 ++-
 .../distributed/dht/GridDhtLockFuture.java      |    4 +-
 .../dht/GridDhtPartitionTopology.java           |   10 +-
 .../dht/GridDhtPartitionTopologyImpl.java       |  335 +-
 .../dht/GridDhtPartitionsEvictor.java           |  140 +
 .../dht/GridDhtPartitionsReservation.java       |    6 +-
 .../dht/GridDhtTopologyFutureAdapter.java       |    2 +-
 .../distributed/dht/GridDhtTxLocalAdapter.java  |   14 +-
 .../distributed/dht/GridDhtTxPrepareFuture.java |    5 +
 .../dht/GridPartitionedGetFuture.java           |   13 +-
 .../dht/GridPartitionedSingleGetFuture.java     |   18 +-
 .../dht/atomic/GridDhtAtomicCache.java          |   13 +-
 .../atomic/GridNearAtomicFullUpdateRequest.java |    1 +
 .../colocated/GridDhtColocatedLockFuture.java   |    4 +-
 .../colocated/GridDhtDetachedCacheEntry.java    |   19 +-
 .../CachePartitionFullCountersMap.java          |   22 +
 .../CachePartitionPartialCountersMap.java       |   61 +-
 .../GridDhtPartitionDemandLegacyMessage.java    |  435 ++
 .../GridDhtPartitionDemandMessage.java          |  201 +-
 .../dht/preloader/GridDhtPartitionDemander.java |  445 +-
 .../preloader/GridDhtPartitionExchangeId.java   |    3 +
 .../dht/preloader/GridDhtPartitionSupplier.java |  548 ++-
 .../GridDhtPartitionSupplyMessage.java          |   22 +-
 .../GridDhtPartitionsExchangeFuture.java        |  446 +-
 .../dht/preloader/GridDhtPreloader.java         |  198 +-
 .../IgniteDhtDemandedPartitionsMap.java         |  218 +
 .../IgniteDhtPartitionCountersMap.java          |    6 +
 .../IgniteDhtPartitionHistorySuppliersMap.java  |    6 +
 .../IgniteDhtPartitionsToReloadMap.java         |   13 +
 .../dht/preloader/IgniteHistoricalIterator.java |   39 +
 .../preloader/IgniteRebalanceIteratorImpl.java  |  201 +
 ...StopCachesOnClientReconnectExchangeTask.java |   59 +
 .../distributed/near/GridNearCacheEntry.java    |  220 +-
 .../distributed/near/GridNearGetFuture.java     |   13 +-
 .../distributed/near/GridNearLockFuture.java    |    4 +-
 .../near/GridNearOptimisticTxPrepareFuture.java |   61 +-
 ...ridNearOptimisticTxPrepareFutureAdapter.java |    2 +-
 .../near/GridNearTxPrepareFutureAdapter.java    |    2 +-
 .../distributed/near/IgniteTxMappingsImpl.java  |    4 +-
 .../cache/local/GridLocalCacheEntry.java        |   50 +-
 .../local/atomic/GridLocalAtomicCache.java      |    9 +-
 .../cache/persistence/AllocatedPageTracker.java |   30 +
 .../cache/persistence/CacheDataRow.java         |    2 +-
 .../cache/persistence/CheckpointFuture.java     |   35 +
 .../CheckpointWriteProgressSupplier.java        |   45 +
 .../persistence/DataRegionMetricsImpl.java      |  100 +-
 .../DataRegionMetricsMXBeanImpl.java            |   25 +
 .../persistence/DataRegionMetricsSnapshot.java  |   45 +
 .../cache/persistence/DataStructure.java        |   17 +-
 .../GridCacheDatabaseSharedManager.java         | 1660 ++++++--
 .../persistence/GridCacheOffheapManager.java    |  617 ++-
 .../IgniteCacheDatabaseSharedManager.java       |  136 +-
 .../cache/persistence/IndexStorage.java         |   51 +
 .../cache/persistence/IndexStorageImpl.java     |  420 ++
 .../processors/cache/persistence/MetaStore.java |   51 -
 .../cache/persistence/MetadataStorage.java      |  420 --
 .../cache/persistence/RecoveryDebug.java        |  170 +
 .../processors/cache/persistence/Storable.java  |   38 +
 .../cache/persistence/file/FileDownloader.java  |  189 +
 .../cache/persistence/file/FileIO.java          |    7 +-
 .../cache/persistence/file/FilePageStore.java   |  139 +-
 .../persistence/file/FilePageStoreFactory.java  |    3 +-
 .../persistence/file/FilePageStoreManager.java  |  275 +-
 .../cache/persistence/file/FilePageStoreV2.java |   11 +-
 .../cache/persistence/file/FileUploader.java    |  100 +
 .../file/FileVersionCheckingFactory.java        |   51 +-
 .../file/PersistentStorageIOException.java      |   47 +
 .../cache/persistence/file/UnzipFileIO.java     |  141 +
 .../filename/PdsConsistentIdProcessor.java      |    3 +
 .../persistence/freelist/AbstractFreeList.java  |  603 +++
 .../persistence/freelist/CacheFreeListImpl.java |   59 +
 .../cache/persistence/freelist/FreeList.java    |    8 +-
 .../persistence/freelist/FreeListImpl.java      |  619 ---
 .../cache/persistence/freelist/PagesList.java   |   55 +-
 .../persistence/metastorage/MetaStorage.java    |  547 +++
 .../metastorage/MetastorageDataRow.java         |   91 +
 .../MetastorageLifecycleListener.java           |   47 +
 .../metastorage/MetastorageRowStore.java        |   96 +
 .../metastorage/MetastorageSearchRow.java       |   38 +
 .../metastorage/MetastorageTree.java            |  266 ++
 .../metastorage/MetsatorageSearchRowImpl.java   |   55 +
 .../metastorage/ReadOnlyMetastorage.java        |   40 +
 .../metastorage/ReadWriteMetastorage.java       |   32 +
 .../pagemem/CheckpointMetricsTracker.java       |    2 +-
 .../pagemem/DelayedDirtyPageWrite.java          |  105 +
 .../pagemem/DelayedPageReplacementTracker.java  |  198 +
 .../persistence/pagemem/EvictCandidate.java     |   77 -
 .../persistence/pagemem/FullPageIdTable.java    |  358 +-
 .../pagemem/IntervalBasedMeasurement.java       |  304 ++
 .../persistence/pagemem/LoadedPagesMap.java     |  137 +
 .../cache/persistence/pagemem/PageMemoryEx.java |   36 +-
 .../persistence/pagemem/PageMemoryImpl.java     |  736 ++--
 .../pagemem/PagesWriteSpeedBasedThrottle.java   |  529 +++
 .../persistence/pagemem/PagesWriteThrottle.java |   83 +-
 .../pagemem/PagesWriteThrottlePolicy.java       |   39 +
 .../persistence/pagemem/ReplaceCandidate.java   |   78 +
 .../persistence/pagemem/ReplacedPageWriter.java |   35 +
 .../pagemem/RobinHoodBackwardShiftHashMap.java  |  657 +++
 .../partstate/PagesAllocationRange.java         |    7 +-
 .../partstate/PartitionAllocationMap.java       |   36 +-
 .../snapshot/IgniteCacheSnapshotManager.java    |   23 +-
 .../snapshot/SnapshotDiscoveryMessage.java      |    7 +-
 .../cache/persistence/tree/BPlusTree.java       |   15 +
 .../persistence/tree/io/AbstractDataPageIO.java | 1256 ++++++
 .../cache/persistence/tree/io/DataPageIO.java   | 1214 +-----
 .../cache/persistence/tree/io/PageIO.java       |   26 +-
 .../tree/io/PagePartitionCountersIO.java        |   34 +-
 .../tree/io/PagePartitionMetaIO.java            |   17 +-
 .../persistence/tree/io/SimpleDataPageIO.java   |  127 +
 .../cache/persistence/tree/reuse/ReuseBag.java  |    5 +
 .../persistence/tree/util/PageHandler.java      |    2 +-
 .../wal/AbstractWalRecordsIterator.java         |   97 +-
 .../wal/FileWriteAheadLogManager.java           |  719 ++--
 .../wal/FsyncModeFileWriteAheadLogManager.java  | 3226 +++++++++++++++
 .../persistence/wal/SegmentArchivedMonitor.java |   64 +
 .../wal/SegmentReservationStorage.java          |   61 +
 .../SingleSegmentLogicalRecordsIterator.java    |   12 +-
 .../wal/reader/IgniteWalIteratorFactory.java    |   42 +-
 .../wal/reader/StandaloneGridKernalContext.java |   18 +-
 .../reader/StandaloneWalRecordsIterator.java    |   78 +-
 .../BaselineTopologyRecordSerializer.java       |  168 +
 .../wal/serializer/RecordDataV1Serializer.java  |   80 +-
 .../wal/serializer/RecordDataV2Serializer.java  |  224 +-
 .../wal/serializer/RecordV1Serializer.java      |    6 +-
 .../wal/serializer/RecordV2Serializer.java      |   11 +-
 .../wal/serializer/TxRecordSerializer.java      |  152 +-
 .../query/GridCacheDistributedQueryManager.java |    6 +-
 .../cache/query/GridCacheQueryManager.java      |  331 +-
 .../query/GridCacheQueryMetricsAdapter.java     |   10 +-
 .../cache/query/SqlFieldsQueryEx.java           |   45 +
 .../continuous/CacheContinuousQueryEntry.java   |    9 +-
 .../CacheContinuousQueryEventBuffer.java        |    5 +-
 .../continuous/CacheContinuousQueryHandler.java |  153 +-
 .../CacheContinuousQueryHandlerV2.java          |    9 +-
 .../CacheContinuousQueryHandlerV3.java          |  185 +
 .../continuous/CacheContinuousQueryManager.java |   56 +-
 .../processors/cache/store/CacheLocalStore.java |    4 +
 .../store/GridCacheStoreManagerAdapter.java     |    6 +
 .../cache/store/GridCacheWriteBehindStore.java  |   11 +-
 .../cache/transactions/IgniteTxAdapter.java     |   73 +-
 .../transactions/IgniteTxLocalAdapter.java      |   40 +-
 .../cache/transactions/IgniteTxManager.java     |   21 +-
 .../CollectConflictPartitionKeysTask.java       |  235 ++
 .../cache/verify/PartitionEntryHashRecord.java  |  237 ++
 .../cache/verify/PartitionHashRecord.java       |  130 +
 .../processors/cache/verify/PartitionKey.java   |  103 +
 .../RetrieveConflictPartitionValuesTask.java    |  204 +
 .../verify/VerifyBackupPartitionsTask.java      |  359 ++
 .../processors/cluster/BaselineTopology.java    |  484 +++
 .../cluster/BaselineTopologyHistory.java        |  127 +
 .../cluster/BaselineTopologyHistoryItem.java    |   79 +
 .../processors/cluster/BranchingPointType.java  |   48 +
 .../cluster/ChangeGlobalStateFinishMessage.java |   19 +-
 .../cluster/ChangeGlobalStateMessage.java       |   38 +-
 .../cluster/DiscoveryDataClusterState.java      |  127 +-
 .../cluster/GridClusterStateProcessor.java      |  781 +++-
 .../cluster/IGridClusterStateProcessor.java     |  122 +
 .../continuous/GridContinuousBatchAdapter.java  |    5 +-
 .../continuous/GridContinuousProcessor.java     |   16 +-
 .../datastreamer/DataStreamerImpl.java          |   57 +-
 .../AtomicDataStructureProxy.java               |    4 +-
 .../datastructures/DataStructuresProcessor.java |  109 +-
 .../processors/hadoop/HadoopClassLoader.java    |    5 +-
 .../processors/igfs/IgfsClientSession.java      |    4 +-
 .../processors/igfs/IgfsDataManager.java        |   10 +-
 .../internal/processors/igfs/IgfsImpl.java      |    4 +-
 .../processors/igfs/IgfsLocalMetrics.java       |   30 +-
 .../internal/processors/igfs/IgfsProcessor.java |    4 +-
 .../processors/job/GridJobProcessor.java        |   18 +-
 .../jobmetrics/GridJobMetricsProcessor.java     |    4 +-
 .../GridMarshallerMappingProcessor.java         |    6 +-
 .../odbc/ClientListenerBufferedParser.java      |    6 +-
 .../odbc/ClientListenerNioListener.java         |  113 +-
 .../odbc/ClientListenerNioServerBuffer.java     |   12 +-
 .../odbc/ClientListenerProcessor.java           |   88 +-
 .../odbc/jdbc/JdbcBatchExecuteRequest.java      |   40 +-
 .../odbc/jdbc/JdbcBulkLoadAckResult.java        |  111 +
 .../odbc/jdbc/JdbcBulkLoadBatchRequest.java     |  183 +
 .../odbc/jdbc/JdbcBulkLoadProcessor.java        |  144 +
 .../odbc/jdbc/JdbcConnectionContext.java        |   13 +-
 .../processors/odbc/jdbc/JdbcRequest.java       |    7 +
 .../odbc/jdbc/JdbcRequestHandler.java           |  272 +-
 .../processors/odbc/jdbc/JdbcResult.java        |    8 +
 .../odbc/odbc/OdbcRequestHandler.java           |   90 +-
 .../offheap/GridOffHeapProcessor.java           |    6 +-
 .../platform/PlatformContextImpl.java           |   20 +-
 .../platform/PlatformProcessorImpl.java         |   83 +-
 .../platform/cache/PlatformCache.java           |    1 +
 .../cache/ClientCacheSqlFieldsQueryRequest.java |    8 +-
 .../platform/cluster/PlatformClusterGroup.java  |   15 +-
 .../utils/PlatformConfigurationUtils.java       |   79 +-
 .../platform/utils/PlatformUtils.java           |   51 +-
 .../processors/query/GridQueryIndexing.java     |   44 +-
 .../processors/query/GridQueryProcessor.java    |  178 +-
 .../processors/query/IgniteSQLException.java    |    9 +
 .../internal/processors/query/QueryUtils.java   |    6 +-
 .../processors/query/SqlClientContext.java      |  223 +
 .../processors/resource/GridResourceIoc.java    |    4 +-
 .../processors/rest/GridRestProcessor.java      |    6 +-
 .../handlers/cache/GridCacheCommandHandler.java |  148 +-
 .../cluster/GridChangeStateCommandHandler.java  |    4 +-
 .../top/GridTopologyCommandHandler.java         |    2 +-
 .../rest/protocols/GridRestProtocolAdapter.java |    5 +-
 .../rest/protocols/tcp/GridTcpRestParser.java   |   15 +-
 .../rest/protocols/tcp/GridTcpRestProtocol.java |   31 +-
 .../rest/request/GridRestCacheRequest.java      |   40 +-
 .../service/GridServiceProcessor.java           |   13 +-
 .../processors/service/GridServiceProxy.java    |    6 +-
 .../session/GridTaskSessionProcessor.java       |    4 +-
 .../GridInternalSubscriptionProcessor.java      |   57 +
 .../processors/task/GridTaskProcessor.java      |    4 +-
 .../processors/task/GridTaskWorker.java         |    4 +-
 .../apache/ignite/internal/sql/SqlKeyword.java  |   53 +-
 .../apache/ignite/internal/sql/SqlParser.java   |   71 +-
 .../ignite/internal/sql/SqlParserUtils.java     |   22 +
 .../sql/command/SqlAlterTableCommand.java       |  114 +
 .../sql/command/SqlBulkLoadCommand.java         |  279 ++
 .../sql/command/SqlSetStreamingCommand.java     |  191 +
 .../ignite/internal/util/Base64Encoder.java     |   33 -
 .../ignite/internal/util/Base64EncoderImpl.java |   45 -
 .../internal/util/GridAnnotationsCache.java     |    6 +-
 .../internal/util/GridClassLoaderCache.java     |    4 +-
 .../internal/util/GridConcurrentFactory.java    |    4 +-
 .../internal/util/GridConcurrentHashSet.java    |   10 +-
 .../ignite/internal/util/GridIntList.java       |    2 +-
 .../ignite/internal/util/GridKeyLock.java       |    4 +-
 .../ignite/internal/util/GridLogThrottle.java   |    4 +-
 .../internal/util/GridSynchronizedMap.java      |    4 +-
 .../apache/ignite/internal/util/GridUnsafe.java |   36 +-
 .../ignite/internal/util/IgniteUtils.java       |  213 +-
 .../internal/util/LegacyBase64Encoder.java      |   42 -
 .../apache/ignite/internal/util/MpscQueue.java  |  240 --
 .../ignite/internal/util/StripedExecutor.java   |  304 +-
 .../util/lang/GridFilteredClosableIterator.java |   37 +
 .../util/lang/GridFilteredIterator.java         |    2 +-
 .../ignite/internal/util/lang/GridFunc.java     |   50 +-
 .../ConcurrentDequeFactoryCallable.java         |   40 -
 .../gridfunc/ConcurrentMapFactoryCallable.java  |    4 +-
 .../ignite/internal/util/nio/GridNioServer.java |   48 +-
 .../util/nio/GridNioServerListener.java         |    8 +
 .../util/nio/GridNioServerListenerAdapter.java  |    5 +
 .../util/nio/GridSelectorNioSessionImpl.java    |    5 +-
 .../util/offheap/unsafe/GridUnsafeMap.java      |    8 +-
 .../unsafe/GridUnsafePartitionedMap.java        |    4 +-
 .../internal/visor/VisorMultiNodeTask.java      |    6 +
 .../visor/baseline/VisorBaselineNode.java       |   89 +
 .../visor/baseline/VisorBaselineOperation.java  |   49 +
 .../visor/baseline/VisorBaselineTask.java       |  224 ++
 .../visor/baseline/VisorBaselineTaskArg.java    |   99 +
 .../visor/baseline/VisorBaselineTaskResult.java |  144 +
 .../visor/cache/VisorMemoryMetrics.java         |   74 +
 .../visor/compute/VisorGatewayTask.java         |   16 +-
 .../node/VisorClientConnectorConfiguration.java |  117 +-
 .../node/VisorDataStorageConfiguration.java     |   47 +-
 .../internal/visor/query/VisorQueryTask.java    |   14 +-
 .../internal/visor/query/VisorQueryUtils.java   |    9 +
 .../internal/visor/util/VisorTaskUtils.java     |    8 +-
 .../apache/ignite/logger/java/JavaLogger.java   |    2 +-
 .../ignite/marshaller/MarshallerContext.java    |   14 +
 .../ignite/marshaller/MarshallerUtils.java      |    6 +
 .../ignite/marshaller/jdk/JdkMarshaller.java    |   20 +-
 .../jdk/JdkMarshallerObjectInputStream.java     |    9 +-
 .../org/apache/ignite/mxbean/IgniteMXBean.java  |   25 +
 .../extensions/communication/MessageWriter.java |   10 +
 .../org/apache/ignite/spi/IgniteSpiAdapter.java |    6 +
 .../org/apache/ignite/spi/IgniteSpiContext.java |    7 +
 .../jobstealing/JobStealingCollisionSpi.java    |   10 +-
 .../tcp/TcpCommunicationMetricsListener.java    |  312 +-
 .../communication/tcp/TcpCommunicationSpi.java  |   18 +-
 .../deployment/local/LocalDeploymentSpi.java    |    4 +-
 .../ignite/spi/discovery/tcp/ClientImpl.java    |   27 +-
 .../ignite/spi/discovery/tcp/ServerImpl.java    |   29 +-
 .../spi/discovery/tcp/TcpDiscoveryImpl.java     |    7 +-
 .../spi/discovery/tcp/TcpDiscoverySpi.java      |   64 +-
 .../tcp/internal/DiscoveryDataPacket.java       |   14 +-
 .../tcp/internal/TcpDiscoveryNode.java          |    7 +-
 .../adaptive/AdaptiveLoadBalancingSpi.java      |    4 +-
 .../roundrobin/RoundRobinLoadBalancingSpi.java  |    4 +-
 .../WeightedRandomLoadBalancingSpi.java         |    4 +-
 .../org/apache/ignite/stream/StreamAdapter.java |    2 +-
 .../apache/ignite/stream/StreamTransformer.java |   60 +-
 .../ignite/stream/socket/SocketStreamer.java    |    8 +-
 .../apache/ignite/util/deque/FastSizeDeque.java |  372 ++
 .../java/org/jsr166/ConcurrentHashMap8.java     | 3808 ------------------
 .../java/org/jsr166/ConcurrentLinkedDeque8.java |    3 +-
 .../org/jsr166/ConcurrentLinkedHashMap.java     |    5 +-
 .../src/main/java/org/jsr166/LongAdder8.java    |  211 -
 .../src/main/java/org/jsr166/Striped64_8.java   |  350 --
 .../java/org/jsr166/ThreadLocalRandom8.java     |  207 -
 .../resources/META-INF/classnames.properties    |  107 +-
 .../core/src/main/resources/ignite.properties   |    2 +-
 .../test/config/class_list_exploit_excluded.txt |   18 +
 .../test/config/class_list_exploit_included.txt |   19 +
 modules/core/src/test/config/log4j2-test.xml    |    6 +-
 .../src/test/config/log4j2-verbose-test.xml     |    4 +-
 ...CacheJdbcBlobStoreMultithreadedSelfTest.java |    6 +-
 .../apache/ignite/internal/ClassSetTest.java    |   71 +
 .../ignite/internal/GridVersionSelfTest.java    |    2 -
 .../IgniteClientReconnectApiExceptionTest.java  |   26 +-
 .../IgniteClientReconnectCacheTest.java         |    6 +-
 .../IgniteReflectionFactorySelfTest.java        |  235 ++
 ...UpdateNotifierPerClusterSettingSelfTest.java |    2 -
 .../MarshallerContextLockingSelfTest.java       |    2 +-
 .../internal/TestDelayingCommunicationSpi.java  |    4 +-
 .../binary/BinaryMarshallerSelfTest.java        |    3 +-
 .../binary/BinaryObjectExceptionSelfTest.java   |   41 +
 .../internal/binary/BinaryTreeSelfTest.java     |  273 +-
 ...GridBinaryMarshallerCtxDisabledSelfTest.java |   12 +
 .../commandline/CommandHandlerParsingTest.java  |  118 +
 .../GridDeploymentMessageCountSelfTest.java     |    4 +-
 .../OptimizedObjectStreamSelfTest.java          |   96 +-
 .../pagemem/impl/PageMemoryNoLoadSelfTest.java  |    2 +-
 .../cache/CacheMetricsEnableRuntimeTest.java    |    4 +-
 .../cache/CacheMetricsEntitiesCountTest.java    |  172 +
 .../cache/CacheNearReaderUpdateTest.java        |    4 +-
 .../CacheSerializableTransactionsTest.java      |    4 +-
 .../DataStorageConfigurationValidationTest.java |   46 +
 .../cache/GridCacheAbstractFullApiSelfTest.java |   11 +-
 .../GridCacheAbstractRemoveFailureTest.java     |   80 +-
 .../cache/GridCacheAbstractSelfTest.java        |    3 +-
 .../cache/GridCacheEntryMemorySizeSelfTest.java |   11 +-
 .../GridCacheInterceptorAbstractSelfTest.java   |   12 +-
 .../GridCacheOrderedPreloadingSelfTest.java     |    8 +-
 ...ridCacheReplicatedSynchronousCommitTest.java |    4 +-
 ...ridCacheStoreManagerDeserializationTest.java |    4 +-
 .../processors/cache/GridCacheTestEntryEx.java  |   19 +-
 .../cache/IgniteCacheAbstractTest.java          |    6 +-
 .../IgniteCacheReadThroughStoreCallTest.java    |    4 +-
 ...IgniteClientCacheInitializationFailTest.java |   20 +-
 .../IgniteClusterActivateDeactivateTest.java    |  251 +-
 ...erActivateDeactivateTestWithPersistence.java |   79 +-
 .../cache/IgniteDynamicCacheStartSelfTest.java  |    8 +
 .../IgniteMarshallerCacheFSRestoreTest.java     |    2 +-
 .../IgniteOnePhaseCommitNearReadersTest.java    |    2 +-
 .../processors/cache/MapCacheStoreStrategy.java |    4 +-
 .../cache/WalModeChangeAbstractSelfTest.java    |  282 ++
 .../cache/WalModeChangeAdvancedSelfTest.java    |  533 +++
 .../WalModeChangeCommonAbstractSelfTest.java    |  328 ++
 ...hangeCoordinatorNotAffinityNodeSelfTest.java |   30 +
 .../processors/cache/WalModeChangeSelfTest.java |   30 +
 ...naryObjectsAbstractDataStreamerSelfTest.java |   14 +-
 ...aryObjectsAbstractMultiThreadedSelfTest.java |   18 +-
 .../GridCacheBinaryStoreAbstractSelfTest.java   |    4 +-
 ...eAbstractDataStructuresFailoverSelfTest.java |   77 +
 .../IgniteClientDataStructuresAbstractTest.java |   40 +-
 .../distributed/Cache64kPartitionsTest.java     |    2 +-
 .../distributed/CacheBaselineTopologyTest.java  |  950 +++++
 .../distributed/CacheExchangeMergeTest.java     |   10 +-
 .../CacheLateAffinityAssignmentTest.java        |    5 +-
 .../CachePutAllFailoverAbstractTest.java        |   35 +-
 .../distributed/GridCacheEventAbstractTest.java |   82 +-
 ...chePartitionedReloadAllAbstractSelfTest.java |    4 +-
 .../distributed/IgniteCache150ClientsTest.java  |    4 +-
 .../IgniteCachePartitionLossPolicySelfTest.java |   31 +-
 .../IgniteOptimisticTxSuspendResumeTest.java    |   52 +
 ...achePartitionPartialCountersMapSelfTest.java |   57 +
 .../dht/GridCacheGlobalLoadTest.java            |    6 +-
 ...ridCachePartitionedUnloadEventsSelfTest.java |   25 +-
 .../dht/IgniteCrossCacheTxSelfTest.java         |    8 +-
 ...cingDelayedPartitionMapExchangeSelfTest.java |    4 +-
 ...idCacheRebalancingPartitionCountersTest.java |  180 +
 .../GridCacheRebalancingSyncSelfTest.java       |    3 +-
 ...idCacheRebalancingWithAsyncClearingTest.java |  240 ++
 ...ridCacheReplicatedEventDisabledSelfTest.java |   75 +
 .../GridCacheReplicatedPreloadSelfTest.java     |    7 +
 .../eviction/paged/PageEvictionMetricTest.java  |   69 +
 .../PageEvictionMultinodeMixedRegionsTest.java  |    9 +-
 .../SortedEvictionPolicyPerformanceTest.java    |    8 +-
 .../IgniteCacheLoadAllAbstractTest.java         |    6 +-
 .../IgniteCacheLoaderWriterAbstractTest.java    |    4 +-
 ...eBaselineAffinityTopologyActivationTest.java |  993 +++++
 .../IgniteDataStorageMetricsSelfTest.java       |    8 +-
 ...tePdsBinaryMetadataOnClusterRestartTest.java |  272 +-
 .../IgnitePdsBinarySortObjectFieldsTest.java    |    6 +-
 ...gnitePdsCacheAssignmentNodeRestartsTest.java |    4 +-
 .../IgnitePdsCacheRebalancingAbstractTest.java  |   89 +-
 .../IgnitePdsClientNearCachePutGetTest.java     |   33 +-
 .../IgnitePdsContinuousRestartTest.java         |   13 +-
 .../IgnitePdsContinuousRestartTest2.java        |  281 ++
 .../IgnitePdsCorruptedStoreTest.java            |  154 +
 .../persistence/IgnitePdsDynamicCacheTest.java  |   15 +-
 .../IgnitePdsExchangeDuringCheckpointTest.java  |    6 +-
 ...MarshallerMappingRestoreOnNodeStartTest.java |    5 +-
 .../IgnitePdsMultiNodePutGetRestartTest.java    |    6 +-
 .../persistence/IgnitePdsPageSizesTest.java     |    7 +-
 ...gnitePdsRecoveryAfterFileCorruptionTest.java |   13 +-
 .../IgnitePdsRemoveDuringRebalancingTest.java   |    4 +-
 ...gnitePdsSingleNodePutGetPersistenceTest.java |   22 +-
 .../IgnitePersistentStoreCacheGroupsTest.java   |    6 +-
 ...IgnitePersistentStoreDataStructuresTest.java |    6 +-
 ...teAllBaselineNodesOnlineFullApiSelfTest.java |   35 +
 .../IgniteBaselineAbstractFullApiSelfTest.java  |   52 +
 ...gingBaselineDownCachePutAllFailoverTest.java |  127 +
 ...gingBaselineDownCacheRemoveFailoverTest.java |  157 +
 ...angingBaselineUpCachePutAllFailoverTest.java |  122 +
 ...angingBaselineUpCacheRemoveFailoverTest.java |  154 +
 ...gniteOfflineBaselineNodeFullApiSelfTest.java |   39 +
 ...eOnlineNodeOutOfBaselineFullApiSelfTest.java |   37 +
 ...teStableBaselineCachePutAllFailoverTest.java |  105 +
 ...teStableBaselineCacheRemoveFailoverTest.java |  131 +
 .../db/IgnitePdsCacheRestoreTest.java           |    6 +-
 .../db/IgnitePdsDataRegionMetricsTest.java      |  232 ++
 .../db/IgnitePdsMultiNodePutGetRestartTest.java |    6 +-
 ...PdsPageEvictionDuringPartitionClearTest.java |    6 +-
 .../db/IgnitePdsPageEvictionTest.java           |    7 +-
 ...tePdsRebalancingOnNotStableTopologyTest.java |    6 +-
 .../db/IgnitePdsTransactionsHangTest.java       |    9 +-
 .../db/IgnitePdsWholeClusterRestartTest.java    |   16 +-
 ...gniteCheckpointDirtyPagesForLowLoadTest.java |  205 +
 .../checkpoint/IgniteMassLoadSandboxTest.java   |  517 +++
 .../db/checkpoint/ProgressWatchdog.java         |  495 +++
 ...faultPageSizeBackwardsCompatibilityTest.java |   14 +-
 ...nitePdsCacheDestroyDuringCheckpointTest.java |  154 +
 .../db/file/IgnitePdsCacheIntegrationTest.java  |    7 +-
 .../db/file/IgnitePdsCheckpointSimpleTest.java  |  100 +
 ...ckpointSimulationWithRealCpDisabledTest.java |   99 +-
 .../file/IgnitePdsDiskErrorsRecoveringTest.java |  362 ++
 .../db/file/IgnitePdsEvictionTest.java          |  331 --
 .../file/IgnitePdsNoActualWalHistoryTest.java   |    7 +-
 .../db/file/IgnitePdsPageReplacementTest.java   |  329 ++
 .../file/IgnitePdsThreadInterruptionTest.java   |  157 +-
 .../IgniteUidAsConsistentIdMigrationTest.java   |   11 +-
 .../db/wal/IgniteWalFlushDefaultSelfTest.java   |    2 +-
 .../db/wal/IgniteWalFlushFailoverTest.java      |   40 +-
 ...lFlushMultiNodeFailoverAbstractSelfTest.java |   38 +-
 .../wal/IgniteWalHistoryReservationsTest.java   |   70 +-
 .../db/wal/IgniteWalRecoveryPPCTest.java        |    4 +-
 .../IgniteWalRecoverySeveralRestartsTest.java   |    6 +-
 .../db/wal/IgniteWalRecoveryTest.java           |  288 +-
 .../db/wal/IgniteWalSerializerVersionTest.java  |   42 +-
 .../persistence/db/wal/WalCompactionTest.java   |   12 +-
 .../db/wal/WalRecoveryTxLogicalRecordsTest.java |  121 +-
 .../db/wal/reader/IgniteWalReaderTest.java      |   50 +-
 .../db/wal/reader/MockWalIteratorFactory.java   |    2 +-
 .../persistence/file/FileDownloaderTest.java    |   90 +
 .../metastorage/IgniteMetaStorageBasicTest.java |  147 +
 .../pagemem/BPlusTreePageMemoryImplTest.java    |   20 +-
 .../BPlusTreeReuseListPageMemoryImplTest.java   |   20 +-
 .../pagemem/FillFactorMetricTest.java           |  214 +
 .../pagemem/FullPageIdTableTest.java            |  195 +-
 ...gnitePageMemReplaceDelayedWriteUnitTest.java |  306 ++
 .../pagemem/IgniteThrottlingUnitTest.java       |  273 ++
 .../pagemem/IndexStoragePageMemoryImplTest.java |   98 +
 .../MetadataStoragePageMemoryImplTest.java      |  104 -
 .../pagemem/NoOpPageStoreManager.java           |   20 +
 .../persistence/pagemem/NoOpWALManager.java     |   17 +-
 .../pagemem/PageMemoryImplNoLoadTest.java       |   14 +-
 .../persistence/pagemem/PageMemoryImplTest.java |  219 +-
 .../pagemem/PagesWriteThrottleSandboxTest.java  |    8 +-
 .../pagemem/PagesWriteThrottleSmokeTest.java    |   11 +-
 .../RobinHoodBackwardShiftHashMapTest.java      |  442 ++
 .../AbstractNodeJoinTemplate.java               |    6 +-
 .../IgniteChangeGlobalStateAbstractTest.java    |    5 +-
 .../IgniteChangeGlobalStateFailOverTest.java    |    6 +-
 .../IgniteStandByClusterTest.java               |    6 +-
 .../extended/GridActivateExtensionTest.java     |    4 +-
 ...gniteAbstractStandByClientReconnectTest.java |  140 +-
 .../IgniteStandByClientReconnectTest.java       |   94 +-
 ...eStandByClientReconnectToNewClusterTest.java |    6 +-
 .../wal/SegmentedRingByteBufferTest.java        |  188 +-
 .../GridCacheQueryTransformerSelfTest.java      |   79 +
 ...CacheContinuousQueryCounterAbstractTest.java |    4 +-
 ...acheContinuousQueryRandomOperationsTest.java |  156 +-
 ...ContinuousWithTransformerClientSelfTest.java |   40 +
 ...heContinuousWithTransformerFailoverTest.java |  309 ++
 ...eContinuousWithTransformerLocalSelfTest.java |   29 +
 ...nuousWithTransformerPartitionedSelfTest.java |   29 +
 ...uousWithTransformerRandomOperationsTest.java |   31 +
 ...inuousWithTransformerReplicatedSelfTest.java |  511 +++
 ...ridCacheContinuousQueryAbstractSelfTest.java |   10 +-
 ...eContinuousQueryMultiNodesFilteringTest.java |    3 +-
 .../TxDeadlockDetectionNoHangsTest.java         |    2 +
 .../TxPessimisticDeadlockDetectionTest.java     |    2 +
 .../transactions/TxRollbackOnTimeoutTest.java   |   10 +-
 .../processors/database/BPlusTreeSelfTest.java  |   12 +-
 .../database/CacheFreeListImplSelfTest.java     |  565 +++
 .../database/FreeListImplSelfTest.java          |  565 ---
 .../database/IgniteDbAbstractTest.java          |   29 +-
 .../IgniteDbClientNearCachePutGetTest.java      |   39 -
 .../database/IgniteDbDynamicCacheSelfTest.java  |    9 +-
 .../IgniteDbMemoryLeakAbstractTest.java         |    2 +-
 .../database/IgniteDbPutGetAbstractTest.java    |    1 +
 .../IgniteDbPutGetWithCacheStoreTest.java       |  196 +
 .../database/IndexStorageSelfTest.java          |  171 +
 .../database/MetadataStorageSelfTest.java       |  171 -
 .../database/SwapPathConstructionSelfTest.java  |    4 +-
 .../datastreamer/DataStreamerImplSelfTest.java  |   22 +-
 .../IgniteDataStreamerPerformanceTest.java      |   12 +-
 .../processors/igfs/IgfsSizeSelfTest.java       |   19 +-
 .../cache/GridCacheCommandHandlerSelfTest.java  |    9 +-
 .../processors/service/DummyService.java        |   12 +-
 ...gniteServiceConfigVariationsFullApiTest.java |   58 +-
 .../internal/sql/SqlParserBulkLoadSelfTest.java |   90 +
 .../sql/SqlParserSetStreamingSelfTest.java      |  134 +
 .../internal/util/IgniteDevOnlyLogTest.java     |  113 +
 .../util/future/GridCompoundFutureSelfTest.java |    8 +-
 .../future/GridFutureListenPerformanceTest.java |    8 +-
 .../util/future/GridFutureQueueTest.java        |    5 +-
 .../ipc/shmem/IpcSharedMemorySpaceSelfTest.java |    6 +-
 .../offheap/GridOffHeapMapAbstractSelfTest.java |    4 +-
 ...idOffHeapPartitionedMapAbstractSelfTest.java |    5 +-
 ...apPartitionedMapPerformanceAbstractTest.java |    6 +-
 .../ignite/jvmtest/BlockingQueueTest.java       |    4 +-
 .../ignite/jvmtest/ConcurrentMapTest.java       |  200 -
 .../ignite/lang/GridBasicPerformanceTest.java   |   14 +-
 .../lang/GridFutureListenPerformanceTest.java   |    8 +-
 .../GridCircularBufferPerformanceTest.java      |   15 +-
 .../lang/utils/GridCircularBufferSelfTest.java  |    9 +-
 .../utils/GridConcurrentHashMapSelfTest.java    |  110 -
 .../colocation/GridTestCacheStore.java          |    4 +-
 .../communication/GridIoManagerBenchmark.java   |   14 +-
 .../communication/GridIoManagerBenchmark0.java  |   14 +-
 .../GridContinuousOperationsLoadTest.java       |    4 +-
 .../ignite/loadtests/dsi/GridDsiPerfJob.java    |    6 +-
 ...dBoundedConcurrentLinkedHashSetLoadTest.java |    4 +-
 .../loadtests/hashmap/GridCacheTestContext.java |    2 +
 .../job/GridJobExecutionLoadTestClient.java     |    4 +-
 ...GridJobExecutionLoadTestClientSemaphore.java |    4 +-
 ...JobExecutionSingleNodeSemaphoreLoadTest.java |    6 +-
 ...apPartitionedMapPerformanceAbstractTest.java |    6 +-
 .../marshaller/MarshallerContextSelfTest.java   |   10 +-
 .../marshaller/MarshallerContextTestImpl.java   |    6 +-
 .../messaging/IgniteMessagingSendAsyncTest.java |    4 +-
 .../ignite/p2p/P2PStreamingClassLoaderTest.java |  121 +
 .../platform/PlatformCacheWriteMetricsTask.java |    5 +
 .../tcp/GridTcpCommunicationSpiLanTest.java     |   13 +-
 ...cpCommunicationSpiMultithreadedSelfTest.java |   30 +-
 .../tcp/TcpCommunicationStatisticsTest.java     |   14 +-
 .../IgniteDiscoveryCacheReuseSelfTest.java      |   18 +-
 .../DiscoveryUnmarshalVulnerabilityTest.java    |  180 +
 .../tcp/TcpDiscoverySpiReconnectDelayTest.java  |  446 ++
 .../startup/GridRandomCommandLineLoader.java    |    2 +-
 ...ocketStreamerUnmarshalVulnerabilityTest.java |  222 +
 .../testframework/GridSpiTestContext.java       |    6 +
 .../ignite/testframework/GridTestUtils.java     |   13 +-
 .../junits/GridTestKernalContext.java           |    3 +-
 .../junits/common/GridCommonAbstractTest.java   |  125 +-
 .../multijvm/IgniteClusterProcessProxy.java     |   41 +
 .../ignite/testsuites/IgniteBasicTestSuite.java |   14 +-
 .../IgniteCacheEvictionSelfTestSuite.java       |    3 +
 .../IgniteCacheFailoverTestSuite.java           |    4 +
 .../IgniteCacheFailoverTestSuite2.java          |    4 +
 .../IgniteCacheMetricsSelfTestSuite.java        |    4 +-
 .../ignite/testsuites/IgniteCacheTestSuite.java |    2 +
 .../testsuites/IgniteCacheTestSuite2.java       |    6 +
 .../testsuites/IgniteCacheTestSuite3.java       |    6 +
 .../testsuites/IgniteCacheTestSuite6.java       |    7 +
 .../ignite/testsuites/IgniteIpcTestSuite.java   |   38 +
 .../testsuites/IgniteP2PSelfTestSuite.java      |    2 +
 .../ignite/testsuites/IgnitePdsTestSuite.java   |   66 +-
 .../ignite/testsuites/IgnitePdsTestSuite2.java  |   58 +-
 .../testsuites/IgnitePdsUnitTestSuite.java      |   38 +
 .../testsuites/IgniteReproducingSuite.java      |    6 +-
 .../IgniteSpiDiscoverySelfTestSuite.java        |    6 +
 .../testsuites/IgniteStandByClusterSuite.java   |    5 +
 .../testsuites/IgniteStreamSelfTestSuite.java   |    2 +
 .../testsuites/IgniteUtilSelfTestSuite.java     |   10 +-
 .../ignite/util/GridCommandHandlerTest.java     |  276 ++
 .../util/mbeans/GridMBeanDisableSelfTest.java   |   69 +-
 .../mbeans/GridMBeanExoticNamesSelfTest.java    |   70 +
 modules/dev-utils/pom.xml                       |   28 +-
 .../development/utils/IgniteWalConverter.java   |   32 +-
 .../development/utils/RecordSizeCountStat.java  |   83 +
 .../ignite/development/utils/TxWalStat.java     |  318 ++
 .../ignite/development/utils/WalStat.java       |  328 ++
 modules/direct-io/licenses/apache-2.0.txt       |  202 +
 modules/direct-io/pom.xml                       |  122 +
 .../cache/persistence/file/AlignedBuffers.java  |   72 +
 .../file/AlignedBuffersDirectFileIO.java        |  511 +++
 .../file/AlignedBuffersDirectFileIOFactory.java |  177 +
 .../persistence/file/IgniteNativeIoLib.java     |  405 ++
 .../persistence/file/LinuxNativeIoPlugin.java   |   25 +
 .../file/LinuxNativeIoPluginProvider.java       |  239 ++
 .../org.apache.ignite.plugin.PluginProvider     |    1 +
 .../IgniteNativeIoWithNoPersistenceTest.java    |   76 +
 .../testsuites/IgnitePdsNativeIoTestSuite.java  |   41 +
 .../testsuites/IgnitePdsNativeIoTestSuite2.java |   36 +
 .../IgnitePdsReplacementNativeIoTest.java       |   44 +
 modules/extdata/p2p/pom.xml                     |    2 +-
 .../tests/p2p/NoopCacheEntryProcessor.java      |   36 +
 modules/extdata/platform/pom.xml                |    2 +-
 .../extdata/uri/modules/uri-dependency/pom.xml  |    2 +-
 modules/extdata/uri/pom.xml                     |    2 +-
 modules/flink/pom.xml                           |    2 +-
 modules/flume/pom.xml                           |    2 +-
 modules/gce/pom.xml                             |    2 +-
 modules/geospatial/pom.xml                      |    2 +-
 modules/hadoop/pom.xml                          |    4 +-
 .../hadoop/counter/HadoopCountersImpl.java      |    4 +-
 .../hadoop/impl/fs/HadoopLazyConcurrentMap.java |    4 +-
 .../hadoop/impl/igfs/HadoopIgfsIpcIo.java       |    6 +-
 .../hadoop/impl/igfs/HadoopIgfsOutProc.java     |    5 +-
 .../processors/hadoop/impl/v2/HadoopV2Job.java  |    4 +-
 .../hadoop/jobtracker/HadoopJobTracker.java     |    8 +-
 .../taskexecutor/HadoopExecutorService.java     |    4 +-
 .../external/HadoopExternalTaskExecutor.java    |   10 +-
 .../HadoopExternalCommunication.java            |    4 +-
 .../processors/hadoop/HadoopSharedMap.java      |    6 +-
 .../hadoop/impl/HadoopAbstractSelfTest.java     |   15 +-
 .../hadoop/impl/HadoopCommandLineTest.java      |    4 +-
 .../HadoopIgfs20FileSystemAbstractSelfTest.java |    4 +-
 .../IgniteHadoopFileSystemAbstractSelfTest.java |    4 +-
 .../taskexecutor/HadoopExecutorServiceTest.java |    6 +-
 modules/hibernate-4.2/pom.xml                   |    2 +-
 modules/hibernate-5.1/pom.xml                   |    2 +-
 modules/hibernate-core/pom.xml                  |    2 +-
 modules/ignored-tests/pom.xml                   |    2 +-
 modules/indexing/pom.xml                        |   50 +-
 .../query/h2/DmlStatementsProcessor.java        |  554 ++-
 .../query/h2/H2CachedStatementKey.java          |  108 +
 .../processors/query/h2/H2DmlPlanKey.java       |   81 -
 .../processors/query/h2/H2StatementCache.java   |   28 +-
 .../internal/processors/query/h2/H2Utils.java   |   28 +-
 .../processors/query/h2/IgniteH2Indexing.java   |  773 ++--
 .../processors/query/h2/ParsingResult.java      |  102 +
 .../query/h2/database/H2TreeIndex.java          |   15 +
 .../query/h2/ddl/DdlStatementsProcessor.java    |   79 +-
 .../processors/query/h2/dml/DmlBatchSender.java |  247 +-
 .../processors/query/h2/dml/DmlUtils.java       |   12 +
 .../processors/query/h2/dml/UpdateMode.java     |   11 +-
 .../processors/query/h2/dml/UpdatePlan.java     |   83 +-
 .../query/h2/dml/UpdatePlanBuilder.java         |  101 +-
 .../query/h2/opt/GridH2IndexBase.java           |   10 +
 .../query/h2/opt/GridH2MetaTable.java           |    6 +-
 .../query/h2/opt/GridH2QueryContext.java        |    4 +-
 .../processors/query/h2/opt/GridH2Table.java    |   15 +-
 .../query/h2/sql/GridSqlQueryParser.java        |   93 +-
 .../query/h2/sql/GridSqlQuerySplitter.java      |    3 +-
 .../query/h2/twostep/GridMapQueryExecutor.java  |   16 +-
 .../h2/twostep/GridReduceQueryExecutor.java     |    5 +-
 .../query/h2/twostep/MapNodeResults.java        |    6 +-
 .../query/h2/twostep/MapQueryLazyWorker.java    |   11 +-
 .../query/h2/twostep/MapQueryResult.java        |   24 +-
 .../query/h2/twostep/MapQueryResults.java       |   18 +-
 .../cache/GridCacheCrossCacheQuerySelfTest.java |   90 +-
 ...GridCacheLazyQueryPartitionsReleaseTest.java |    9 +-
 .../cache/GridCacheQuerySimpleBenchmark.java    |    6 +-
 .../IgniteBinaryObjectFieldsQuerySelfTest.java  |    7 +-
 .../cache/IgniteCacheAbstractQuerySelfTest.java |   59 +-
 ...eLockPartitionOnAffinityRunAbstractTest.java |    8 +-
 ...PartitionOnAffinityRunAtomicCacheOpTest.java |   18 +-
 ...niteCacheLockPartitionOnAffinityRunTest.java |   29 +-
 .../cache/IgniteCacheSqlQueryErrorSelfTest.java |  214 +
 ...CheckClusterStateBeforeExecuteQueryTest.java |    8 +-
 .../cache/IgniteErrorOnRebalanceTest.java       |    2 +-
 ...leNodeWithIndexingPutGetPersistenceTest.java |    7 +-
 .../cache/SqlFieldsQuerySelfTest.java           |   53 +
 ...lientQueryReplicatedNodeRestartSelfTest.java |    1 +
 ...stributedPartitionQueryAbstractSelfTest.java |    4 +-
 ...chePartitionedQueryEvtsDisabledSelfTest.java |   30 +
 .../IgniteCacheQueryNodeRestartSelfTest.java    |   56 +-
 ...acheReplicatedQueryEvtsDisabledSelfTest.java |   30 +
 .../IgniteCacheReplicatedQuerySelfTest.java     |   54 +-
 .../cache/index/AbstractSchemaSelfTest.java     |    3 +
 .../cache/index/DynamicColumnsAbstractTest.java |    2 +-
 .../DynamicIndexAbstractBasicSelfTest.java      |   29 +-
 .../index/DynamicIndexAbstractSelfTest.java     |   33 +-
 .../H2DynamicColumnsAbstractBasicSelfTest.java  |   37 +
 .../index/H2DynamicIndexingComplexTest.java     |    4 +-
 .../cache/index/H2DynamicTableSelfTest.java     |   14 +-
 .../cache/index/LongIndexNameTest.java          |   14 +-
 .../index/OptimizedMarshallerIndexNameTest.java |    2 +-
 .../local/IgniteCacheLocalQuerySelfTest.java    |   62 +-
 ...onnectorConfigurationValidationSelfTest.java |   66 +
 .../client/IgniteDataStreamerTest.java          |  118 +
 ...eDbSingleNodeWithIndexingWalRestoreTest.java |    7 +-
 ...oreQueryWithMultipleClassesPerCacheTest.java |   13 +-
 .../IgnitePersistentStoreSchemaLoadTest.java    |   25 +-
 ...ckPartitionOnAffinityRunAtomicCacheTest.java |   96 +
 ...neLockPartitionOnAffinityRunTxCacheTest.java |   87 +
 ...ngBaselineCacheQueryNodeRestartSelfTest.java |  124 +
 ...StableBaselineBinObjFieldsQuerySelfTest.java |   77 +
 ...eBaselineCacheQueryNodeRestartsSelfTest.java |   68 +
 .../query/IgniteCachelessQueriesSelfTest.java   |  420 ++
 .../query/IgniteQueryDedicatedPoolTest.java     |   15 +-
 .../query/IgniteSqlDefaultValueTest.java        |    2 +-
 .../query/IgniteSqlNotNullConstraintTest.java   |    2 +-
 .../query/IgniteSqlSchemaIndexingTest.java      |   11 +-
 ...IgniteSqlSkipReducerOnUpdateDmlSelfTest.java |    4 +-
 .../MultipleStatementsSqlQuerySelfTest.java     |    8 +-
 .../query/SqlPushDownFunctionTest.java          |   78 +
 .../processors/query/SqlSchemaSelfTest.java     |   12 +-
 .../IgniteCacheAffinityRunTestSuite.java        |    4 +
 .../IgniteCacheQuerySelfTestSuite.java          |   28 +-
 .../IgniteCacheQuerySelfTestSuite2.java         |    4 +
 .../IgniteCacheQuerySelfTestSuite3.java         |   13 +
 .../IgniteCacheWithIndexingTestSuite.java       |    3 +
 .../IgnitePdsWithIndexingCoreTestSuite.java     |    4 +
 modules/jcl/pom.xml                             |    2 +-
 modules/jms11/pom.xml                           |    2 +-
 modules/jta/pom.xml                             |    2 +-
 modules/kafka/pom.xml                           |    2 +-
 modules/kubernetes/pom.xml                      |    2 +-
 modules/log4j/pom.xml                           |    2 +-
 .../apache/ignite/logger/log4j/Log4JLogger.java |  100 +-
 modules/log4j/src/test/config/log4j-debug.xml   |   50 +
 modules/log4j/src/test/config/log4j-info.xml    |   50 +
 .../logger/log4j/GridLog4jConfigUpdateTest.java |  163 +
 .../logger/log4j/GridLog4jWatchDelayTest.java   |   49 +
 .../ignite/testsuites/IgniteLog4jTestSuite.java |   35 +-
 modules/log4j2/pom.xml                          |    6 +-
 .../ignite/logger/log4j2/Log4J2Logger.java      |   45 +-
 modules/log4j2/src/test/config/log4j2-debug.xml |   37 +
 modules/log4j2/src/test/config/log4j2-info.xml  |   37 +
 .../log4j2/src/test/config/log4j2-markers.xml   |   38 +
 .../logger/log4j2/Log4j2ConfigUpdateTest.java   |  152 +
 .../logger/log4j2/Log4j2LoggerMarkerTest.java   |  110 +
 .../testsuites/IgniteLog4j2TestSuite.java       |    8 +-
 modules/mesos/pom.xml                           |   13 +-
 modules/ml/README.txt                           |   16 +-
 modules/ml/pom.xml                              |   40 +-
 .../org/apache/ignite/ml/DatasetTrainer.java    |   42 +
 .../main/java/org/apache/ignite/ml/Trainer.java |    2 +
 .../org/apache/ignite/ml/dataset/Dataset.java   |  213 +
 .../ignite/ml/dataset/DatasetBuilder.java       |   49 +
 .../ignite/ml/dataset/DatasetFactory.java       |  387 ++
 .../ml/dataset/PartitionContextBuilder.java     |   58 +
 .../ignite/ml/dataset/PartitionDataBuilder.java |   63 +
 .../apache/ignite/ml/dataset/UpstreamEntry.java |   53 +
 .../dataset/impl/cache/CacheBasedDataset.java   |  168 +
 .../impl/cache/CacheBasedDatasetBuilder.java    |   95 +
 .../ml/dataset/impl/cache/package-info.java     |   22 +
 .../dataset/impl/cache/util/ComputeUtils.java   |  251 ++
 .../util/DatasetAffinityFunctionWrapper.java    |   75 +
 .../impl/cache/util/PartitionDataStorage.java   |   65 +
 .../impl/cache/util/UpstreamCursorAdapter.java  |   68 +
 .../dataset/impl/cache/util/package-info.java   |   22 +
 .../ml/dataset/impl/local/LocalDataset.java     |   88 +
 .../dataset/impl/local/LocalDatasetBuilder.java |  137 +
 .../ml/dataset/impl/local/package-info.java     |   22 +
 .../ignite/ml/dataset/impl/package-info.java    |   22 +
 .../apache/ignite/ml/dataset/package-info.java  |   22 +
 .../ml/dataset/primitive/DatasetWrapper.java    |   63 +
 .../ml/dataset/primitive/SimpleDataset.java     |  216 +
 .../dataset/primitive/SimpleLabeledDataset.java |   39 +
 .../builder/context/EmptyContextBuilder.java    |   39 +
 .../primitive/builder/context/package-info.java |   22 +
 .../builder/data/SimpleDatasetDataBuilder.java  |   76 +
 .../data/SimpleLabeledDatasetDataBuilder.java   |   86 +
 .../primitive/builder/data/package-info.java    |   22 +
 .../dataset/primitive/builder/package-info.java |   22 +
 .../dataset/primitive/context/EmptyContext.java |   28 +
 .../dataset/primitive/context/package-info.java |   22 +
 .../primitive/data/SimpleDatasetData.java       |   69 +
 .../data/SimpleLabeledDatasetData.java          |   79 +
 .../ml/dataset/primitive/data/package-info.java |   22 +
 .../ml/dataset/primitive/package-info.java      |   28 +
 .../apache/ignite/ml/genetic/Chromosome.java    |  107 +
 .../apache/ignite/ml/genetic/CrossOverJob.java  |  142 +
 .../apache/ignite/ml/genetic/CrossOverTask.java |  137 +
 .../apache/ignite/ml/genetic/FitnessJob.java    |  101 +
 .../apache/ignite/ml/genetic/FitnessTask.java   |  103 +
 .../org/apache/ignite/ml/genetic/GAGrid.java    |  454 +++
 .../java/org/apache/ignite/ml/genetic/Gene.java |   86 +
 .../ignite/ml/genetic/IFitnessFunction.java     |   39 +
 .../org/apache/ignite/ml/genetic/MutateJob.java |   94 +
 .../apache/ignite/ml/genetic/MutateTask.java    |  191 +
 .../ignite/ml/genetic/TruncateSelectionJob.java |   88 +
 .../ml/genetic/TruncateSelectionTask.java       |  173 +
 .../ml/genetic/cache/GeneCacheConfig.java       |   52 +
 .../ml/genetic/cache/PopulationCacheConfig.java |   53 +
 .../ignite/ml/genetic/cache/package-info.java   |   21 +
 .../ml/genetic/functions/GAGridFunction.java    |  132 +
 .../ml/genetic/functions/package-info.java      |   21 +
 .../apache/ignite/ml/genetic/package-info.java  |   21 +
 .../genetic/parameter/ChromosomeCriteria.java   |   50 +
 .../ml/genetic/parameter/GAConfiguration.java   |  333 ++
 .../ml/genetic/parameter/GAGridConstants.java   |   37 +
 .../genetic/parameter/ITerminateCriteria.java   |   39 +
 .../ml/genetic/parameter/package-info.java      |   21 +
 .../ignite/ml/genetic/utils/GAGridUtils.java    |  124 +
 .../ignite/ml/genetic/utils/package-info.java   |   21 +
 .../ml/math/isolve/IterativeSolverResult.java   |   64 +
 .../LinSysPartitionDataBuilderOnHeap.java       |   85 +
 .../math/isolve/LinSysPartitionDataOnHeap.java  |   75 +
 .../ml/math/isolve/lsqr/AbstractLSQR.java       |  333 ++
 .../ignite/ml/math/isolve/lsqr/LSQROnHeap.java  |  102 +
 .../math/isolve/lsqr/LSQRPartitionContext.java  |   41 +
 .../ignite/ml/math/isolve/lsqr/LSQRResult.java  |  140 +
 .../ml/math/isolve/lsqr/package-info.java       |   22 +
 .../ignite/ml/math/isolve/package-info.java     |   22 +
 .../ignite/ml/nn/LocalBatchTrainerInput.java    |   42 -
 .../org/apache/ignite/ml/nn/LossFunctions.java  |   47 -
 .../ml/nn/MLPGroupUpdateTrainerCacheInput.java  |   33 +-
 .../ignite/ml/nn/MultilayerPerceptron.java      |    2 +-
 .../AbstractMLPGroupUpdateTrainerInput.java     |    2 +-
 .../distributed/MLPGroupUpdateTrainer.java      |   91 +-
 .../MLPGroupUpdateTrainerDataCache.java         |    2 +-
 .../distributed/MLPGroupUpdateTrainingData.java |   35 +-
 .../MLPGroupUpdateTrainingLoopData.java         |   36 +-
 .../trainers/distributed/MLPMetaoptimizer.java  |    5 +-
 .../nn/trainers/distributed/package-info.java   |   22 +
 .../ml/nn/trainers/local/LocalBatchTrainer.java |  179 -
 .../nn/trainers/local/MLPLocalBatchTrainer.java |   13 +-
 .../ml/nn/updaters/BaseSmoothParametrized.java  |   66 -
 .../ml/nn/updaters/NesterovParameterUpdate.java |   94 -
 .../nn/updaters/NesterovUpdateCalculator.java   |   85 -
 .../nn/updaters/ParameterUpdateCalculator.java  |   58 -
 .../ml/nn/updaters/RPropParameterUpdate.java    |  228 --
 .../ml/nn/updaters/RPropUpdateCalculator.java   |  151 -
 .../ml/nn/updaters/SimpleGDParameter.java       |   77 -
 .../nn/updaters/SimpleGDUpdateCalculator.java   |   66 -
 .../ml/nn/updaters/SmoothParametrized.java      |   27 -
 .../ignite/ml/nn/updaters/package-info.java     |   22 -
 .../ml/optimization/BarzilaiBorweinUpdater.java |    8 +-
 .../ml/optimization/BaseParametrized.java       |   58 +
 .../ignite/ml/optimization/GradientDescent.java |    3 +-
 .../ml/optimization/GradientFunction.java       |    2 +-
 .../ignite/ml/optimization/LossFunctions.java   |   47 +
 .../ignite/ml/optimization/Parametrized.java    |   26 +
 .../ignite/ml/optimization/SimpleUpdater.java   |    3 +-
 .../ml/optimization/SmoothParametrized.java     |   45 +
 .../NesterovParameterUpdate.java                |   94 +
 .../NesterovUpdateCalculator.java               |   86 +
 .../ParameterUpdateCalculator.java              |   59 +
 .../updatecalculators/RPropParameterUpdate.java |  228 ++
 .../RPropUpdateCalculator.java                  |  152 +
 .../SimpleGDParameterUpdate.java                |   89 +
 .../SimpleGDUpdateCalculator.java               |   81 +
 .../updatecalculators/package-info.java         |   22 +
 .../util/SparseDistributedMatrixMapReducer.java |    2 +-
 .../ml/preprocessing/PreprocessingTrainer.java  |   42 +
 .../NormalizationPartitionData.java             |   58 +
 .../NormalizationPreprocessor.java              |   88 +
 .../normalization/NormalizationTrainer.java     |   90 +
 .../normalization/package-info.java             |   22 +
 .../ignite/ml/preprocessing/package-info.java   |   22 +
 .../linear/LinearRegressionLSQRTrainer.java     |   70 +
 .../apache/ignite/ml/structures/Dataset.java    |   31 +-
 .../apache/ignite/ml/structures/DatasetRow.java |    1 -
 .../ignite/ml/structures/LabeledDataset.java    |   43 +-
 .../structures/LabeledDatasetTestTrainPair.java |    4 +-
 .../preprocessing/LabeledDatasetLoader.java     |   31 +-
 .../preprocessing/LabellingMachine.java         |   10 +-
 .../ml/structures/preprocessing/Normalizer.java |    2 +
 .../svm/SVMLinearBinaryClassificationModel.java |  186 +
 .../SVMLinearBinaryClassificationTrainer.java   |  264 ++
 .../ignite/ml/svm/SVMPartitionContext.java      |   28 +
 .../ml/svm/SVMPartitionDataBuilderOnHeap.java   |   86 +
 .../org/apache/ignite/ml/svm/package-info.java  |   22 +
 .../org/apache/ignite/ml/trainers/Trainer.java  |    5 +-
 .../trainers/group/BaseLocalProcessorJob.java   |    3 +-
 .../group/GroupTrainerBaseProcessorTask.java    |    3 +-
 .../group/MetaoptimizerDistributedStep.java     |   13 +-
 .../ml/trainers/group/ResultAndUpdates.java     |    3 +-
 .../ml/trainers/group/UpdateStrategies.java     |   47 +
 .../ml/trainers/group/UpdatesStrategy.java      |   94 +
 .../trainers/group/chain/ComputationsChain.java |    4 +-
 .../ml/trainers/local/LocalBatchTrainer.java    |  178 +
 .../trainers/local/LocalBatchTrainerInput.java  |   42 +
 .../ignite/ml/trainers/local/package-info.java  |   22 +
 .../java/org/apache/ignite/ml/util/Utils.java   |   24 +-
 .../org/apache/ignite/ml/IgniteMLTestSuite.java |   14 +-
 .../org/apache/ignite/ml/LocalModelsTest.java   |   19 +
 .../ignite/ml/dataset/DatasetTestSuite.java     |   45 +
 .../cache/CacheBasedDatasetBuilderTest.java     |  107 +
 .../impl/cache/CacheBasedDatasetTest.java       |  351 ++
 .../impl/cache/util/ComputeUtilsTest.java       |  309 ++
 .../DatasetAffinityFunctionWrapperTest.java     |  110 +
 .../cache/util/PartitionDataStorageTest.java    |   49 +
 .../impl/local/LocalDatasetBuilderTest.java     |   91 +
 .../dataset/primitive/DatasetWrapperTest.java   |   87 +
 .../ml/genetic/GAGridCalculateFitnessTest.java  |  141 +
 .../genetic/GAGridInitializePopulationTest.java |  153 +
 .../ignite/ml/genetic/GAGridTestSuite.java      |   33 +
 .../ml/genetic/PasswordFitnessFunction.java     |   61 +
 .../ignite/ml/knn/LabeledDatasetTest.java       |    1 +
 .../apache/ignite/ml/math/ExternalizeTest.java  |    1 +
 .../ignite/ml/math/MathImplLocalTestSuite.java  |    4 +-
 .../org/apache/ignite/ml/math/TracerTest.java   |   37 +-
 .../ml/math/isolve/lsqr/LSQROnHeapTest.java     |  134 +
 .../ignite/ml/nn/MLPGroupTrainerTest.java       |   38 +-
 .../ignite/ml/nn/MLPLocalTrainerTest.java       |   17 +-
 .../java/org/apache/ignite/ml/nn/MLPTest.java   |    1 +
 .../ml/nn/SimpleMLPLocalBatchTrainerInput.java  |    1 +
 .../ml/nn/performance/MnistDistributed.java     |    8 +-
 .../ignite/ml/nn/performance/MnistLocal.java    |    8 +-
 .../ml/optimization/OptimizationTestSuite.java  |   33 +
 .../preprocessing/PreprocessingTestSuite.java   |   35 +
 .../NormalizationPreprocessorTest.java          |   54 +
 .../normalization/NormalizationTrainerTest.java |   76 +
 .../ml/regressions/RegressionsTestSuite.java    |    4 +-
 .../linear/LinearRegressionLSQRTrainerTest.java |  124 +
 .../ignite/ml/svm/SVMBinaryTrainerTest.java     |   74 +
 .../org/apache/ignite/ml/svm/SVMModelTest.java  |  113 +
 .../org/apache/ignite/ml/svm/SVMTestSuite.java  |   33 +
 .../ml/trainers/group/TestGroupTrainer.java     |    2 +-
 .../ml/src/test/resources/datasets/README.md    |    5 +-
 modules/mqtt/pom.xml                            |    2 +-
 modules/osgi-karaf/pom.xml                      |    2 +-
 modules/osgi-paxlogging/pom.xml                 |    2 +-
 modules/osgi/pom.xml                            |    2 +-
 .../org/apache/ignite/osgi/IgniteOsgiUtils.java |    4 +-
 .../classloaders/ContainerSweepClassLoader.java |    4 +-
 modules/platforms/cpp/common/configure.ac       |    2 +-
 .../include/ignite/common/platform_utils.h      |   17 +
 .../cpp/common/include/ignite/common/utils.h    |    7 +
 .../os/linux/src/common/platform_utils.cpp      |   15 +
 .../os/win/src/common/dynamic_load_os.cpp       |    4 +
 .../common/os/win/src/common/platform_utils.cpp |   15 +
 .../platforms/cpp/common/src/common/utils.cpp   |    9 +
 modules/platforms/cpp/configure.ac              |    2 +-
 modules/platforms/cpp/configure.acrel           |    2 +-
 .../cpp/core-test/config/cache-identity-32.xml  |    5 +-
 .../cpp/core-test/config/cache-query-32.xml     |    6 +-
 .../config/cache-query-continuous-32.xml        |    6 +-
 .../cpp/core-test/config/cache-store-32.xml     |    4 +-
 .../cpp/core-test/config/cache-test-32.xml      |    6 +-
 .../cpp/core-test/config/isolated-32.xml        |    5 +-
 .../core-test/config/persistence-store-32.xml   |    5 +-
 modules/platforms/cpp/core-test/configure.ac    |    2 +-
 .../cpp/core-test/src/cache_query_test.cpp      |   58 +-
 modules/platforms/cpp/core/configure.ac         |    2 +-
 modules/platforms/cpp/examples/configure.ac     |    2 +-
 modules/platforms/cpp/ignite/configure.ac       |    2 +-
 modules/platforms/cpp/odbc-test/Makefile.am     |    6 +-
 .../cpp/odbc-test/config/queries-default.xml    |   10 +-
 .../cpp/odbc-test/config/queries-ssl-32.xml     |   77 +
 .../cpp/odbc-test/config/queries-ssl.xml        |   59 +
 .../cpp/odbc-test/config/queries-test-32.xml    |    7 +-
 .../odbc-test/config/queries-test-default.xml   |   39 -
 .../odbc-test/config/queries-test-noodbc-32.xml |   53 -
 .../odbc-test/config/queries-test-noodbc.xml    |   35 -
 .../cpp/odbc-test/config/queries-test.xml       |    3 +-
 .../platforms/cpp/odbc-test/config/ssl/ca.pem   |   24 +
 .../cpp/odbc-test/config/ssl/client_full.pem    |   84 +
 .../cpp/odbc-test/config/ssl/client_unknown.pem |   50 +
 .../cpp/odbc-test/config/ssl/server.jks         |  Bin 0 -> 4256 bytes
 .../cpp/odbc-test/config/ssl/trust.jks          |  Bin 0 -> 1089 bytes
 .../cpp/odbc-test/include/odbc_test_suite.h     |    8 +-
 .../cpp/odbc-test/include/test_utils.h          |    5 +
 .../cpp/odbc-test/project/vs/odbc-test.vcxproj  |   15 +-
 .../project/vs/odbc-test.vcxproj.filters        |   14 +-
 .../cpp/odbc-test/src/api_robustness_test.cpp   |  101 +-
 .../src/application_data_buffer_test.cpp        |    4 +-
 .../platforms/cpp/odbc-test/src/column_test.cpp |  174 +-
 .../platforms/cpp/odbc-test/src/errors_test.cpp |   90 +-
 .../cpp/odbc-test/src/meta_queries_test.cpp     |   96 +-
 .../cpp/odbc-test/src/odbc_test_suite.cpp       |   67 +-
 .../cpp/odbc-test/src/queries_ssl_test.cpp      |  283 ++
 .../cpp/odbc-test/src/queries_test.cpp          |   18 +-
 .../platforms/cpp/odbc-test/src/row_test.cpp    |    1 +
 .../cpp/odbc-test/src/sql_get_info_test.cpp     |    8 +-
 .../odbc-test/src/sql_test_suite_fixture.cpp    |    4 +
 .../platforms/cpp/odbc-test/src/test_utils.cpp  |    9 +
 modules/platforms/cpp/odbc/Makefile.am          |    5 +-
 modules/platforms/cpp/odbc/include/Makefile.am  |    6 +-
 .../ignite/odbc/app/application_data_buffer.h   |  104 +-
 .../include/ignite/odbc/app/parameter_set.h     |   26 +-
 .../cpp/odbc/include/ignite/odbc/column.h       |    3 +-
 .../cpp/odbc/include/ignite/odbc/common_types.h |    6 +
 .../include/ignite/odbc/config/configuration.h  |  110 +-
 .../cpp/odbc/include/ignite/odbc/connection.h   |    9 +-
 .../cpp/odbc/include/ignite/odbc/message.h      |   12 -
 .../odbc/include/ignite/odbc/query/data_query.h |   13 +-
 .../cpp/odbc/include/ignite/odbc/row.h          |    7 +-
 .../odbc/include/ignite/odbc/socket_client.h    |  109 +
 .../ignite/odbc/ssl/secure_socket_client.h      |  148 +
 .../odbc/include/ignite/odbc/ssl/ssl_bindings.h |  303 ++
 .../odbc/include/ignite/odbc/ssl/ssl_gateway.h  |  151 +
 .../cpp/odbc/include/ignite/odbc/ssl/ssl_mode.h |   55 +
 .../cpp/odbc/include/ignite/odbc/statement.h    |    3 +
 .../include/ignite/odbc/system/socket_client.h  |  151 -
 .../ignite/odbc/system/tcp_socket_client.h      |  151 +
 .../odbc/system/ui/dsn_configuration_window.h   |  113 +-
 .../cpp/odbc/include/ignite/odbc/type_traits.h  |    6 +-
 .../cpp/odbc/install/ignite-odbc-amd64.wxs      |    2 +-
 .../cpp/odbc/install/ignite-odbc-x86.wxs        |    2 +-
 .../odbc/os/linux/src/system/socket_client.cpp  |  360 --
 .../os/linux/src/system/tcp_socket_client.cpp   |  384 ++
 .../odbc/os/win/src/system/socket_client.cpp    |  432 --
 .../os/win/src/system/tcp_socket_client.cpp     |  462 +++
 .../src/system/ui/dsn_configuration_window.cpp  |  315 +-
 .../cpp/odbc/os/win/src/system/ui/window.cpp    |    9 +-
 .../platforms/cpp/odbc/project/vs/odbc.vcxproj  |   36 +-
 .../cpp/odbc/project/vs/odbc.vcxproj.filters    |   39 +-
 .../odbc/src/app/application_data_buffer.cpp    |  545 ++-
 .../cpp/odbc/src/app/parameter_set.cpp          |   21 +-
 modules/platforms/cpp/odbc/src/column.cpp       |   55 +-
 .../cpp/odbc/src/config/configuration.cpp       |    8 +
 .../cpp/odbc/src/config/connection_info.cpp     |  139 +-
 modules/platforms/cpp/odbc/src/connection.cpp   |   74 +-
 .../odbc/src/diagnostic/diagnosable_adapter.cpp |    6 +-
 .../odbc/src/diagnostic/diagnostic_record.cpp   |   12 +
 modules/platforms/cpp/odbc/src/dsn_config.cpp   |   24 +-
 modules/platforms/cpp/odbc/src/message.cpp      |    4 +-
 .../platforms/cpp/odbc/src/meta/column_meta.cpp |   37 +-
 modules/platforms/cpp/odbc/src/odbc.cpp         |   16 +-
 .../cpp/odbc/src/query/batch_query.cpp          |   13 +-
 .../platforms/cpp/odbc/src/query/data_query.cpp |   77 +-
 modules/platforms/cpp/odbc/src/row.cpp          |    9 +-
 .../cpp/odbc/src/ssl/secure_socket_client.cpp   |  419 ++
 .../platforms/cpp/odbc/src/ssl/ssl_gateway.cpp  |  210 +
 modules/platforms/cpp/odbc/src/ssl/ssl_mode.cpp |   43 +
 modules/platforms/cpp/odbc/src/statement.cpp    |   35 +
 modules/platforms/cpp/odbc/src/type_traits.cpp  |   50 +-
 .../Properties/AssemblyInfo.cs                  |    8 +-
 .../Apache.Ignite.AspNet.nuspec                 |    2 +-
 .../Properties/AssemblyInfo.cs                  |    8 +-
 .../Apache.Ignite.Benchmarks.csproj             |    5 +
 .../Apache.Ignite.Benchmarks/BenchmarkRunner.cs |   12 +-
 .../Interop/GetAllBenchmark.cs                  |   71 +
 .../Interop/GetAllBinaryBenchmark.cs            |   58 +
 .../Apache.Ignite.Benchmarks/Model/Doubles.cs   |   69 +
 .../Properties/AssemblyInfo.cs                  |    8 +-
 .../ThinClient/ThinClientGetAllBenchmark.cs     |   72 +
 .../ThinClientGetAllBinaryBenchmark.cs          |   58 +
 .../Apache.Ignite.Core.Tests.DotNetCore.csproj  |   28 +-
 .../Common/TestUtils.DotNetCore.cs              |    3 +-
 .../Properties/AssemblyInfo.cs                  |    8 +-
 .../Properties/AssemblyInfo.cs                  |    8 +-
 .../Apache.Ignite.Core.Tests.csproj             |   56 +-
 .../ApiParity/CacheConfigurationParityTest.cs   |    3 +-
 .../ClientConnectorConfigurationParityTest.cs   |   11 +-
 .../ApiParity/ClusterNodeParityTest.cs          |   45 +
 .../ApiParity/DataRegionMetricsParityTest.cs    |   10 +-
 .../DataStorageConfigurationParityTest.cs       |    2 +-
 .../ApiParity/DataStorageMetricsParityTest.cs   |    2 +-
 .../ApiParity/IgniteConfigurationParityTest.cs  |    3 +-
 .../ApiParity/IgniteParityTest.cs               |    2 -
 .../ApiParity/ParityTest.cs                     |   26 +-
 .../QueryEntityConfigurationParityTest.cs       |    2 +-
 .../Binary/IO/BinaryStreamsTest.cs              |    5 +-
 .../Affinity/AffinityFunctionSpringTest.cs      |    2 +-
 .../Cache/CacheAbstractTest.cs                  |   72 -
 .../Cache/CacheAbstractTransactionalTest.cs     |    5 +
 .../Cache/CacheConfigurationTest.cs             |    2 +-
 .../Cache/CacheCreateTest.cs                    |  152 +
 .../Cache/CacheForkedTest.cs                    |    1 +
 .../Cache/DataRegionMetricsTest.cs              |    6 +
 .../Cache/DataStorageMetricsTest.cs             |    6 +-
 .../Cache/PersistenceTest.cs                    |  166 +-
 .../Cache/PersistentStoreTestObsolete.cs        |    2 +-
 .../Cache/Query/CacheDmlQueriesTest.cs          |    2 +-
 .../Query/CacheQueriesCodeConfigurationTest.cs  |    2 +-
 .../Cache/Query/CacheQueriesTest.cs             |    6 +-
 .../Cache/Query/Linq/CacheLinqTest.Base.cs      |    4 +
 .../Cache/Query/Linq/CacheLinqTest.Custom.cs    |  195 +
 .../Cache/Query/Linq/CacheLinqTest.Functions.cs |   48 +
 .../Cache/Query/Linq/CacheLinqTest.Misc.cs      |    4 +-
 .../Cache/Query/Linq/CacheLinqTest.Numerics.cs  |    4 +-
 .../Client/Cache/CacheClientAsyncWrapper.cs     |    1 -
 .../Client/Cache/CacheTest.cs                   |    3 +-
 .../Client/Cache/CacheTestNoMeta.cs             |    2 +-
 .../Client/Cache/CacheTestSsl.cs                |   65 +
 .../Cache/ClientCacheConfigurationTest.cs       |    2 +-
 .../Client/Cache/CreateCacheTest.cs             |    2 +-
 .../Client/Cache/SqlQueryTest.cs                |    3 +-
 .../Client/ClientConnectionTest.cs              |   67 +-
 .../Client/ClientTestBase.cs                    |    4 +-
 .../Client/IgniteClientConfigurationTest.cs     |    9 +
 .../Client/RawSecureSocketTest.cs               |  159 +
 .../Compute/CancellationTest.cs                 |    2 +-
 .../Compute/ComputeApiTest.cs                   |   10 +-
 .../Config/Apache.Ignite.exe.config.test        |    4 +-
 .../Config/Apache.Ignite.exe.config.test2       |    2 +-
 .../Config/Client/IgniteClientConfiguration.xml |  Bin 2482 -> 3144 bytes
 .../Config/Client/server-with-ssl.xml           |   67 +
 .../Config/Client/server.jks                    |  Bin 0 -> 4256 bytes
 .../Config/Client/thin-client-cert.pfx          |  Bin 0 -> 4549 bytes
 .../Config/Client/trust.jks                     |  Bin 0 -> 1089 bytes
 .../Config/KeyStore/server.jks                  |  Bin 0 -> 3230 bytes
 .../Config/KeyStore/trust.jks                   |  Bin 0 -> 2432 bytes
 .../Config/full-config.xml                      |    5 +-
 .../Apache.Ignite.Core.Tests/Config/ssl.xml     |   65 +
 .../Config/start-test-grid1.xml                 |   55 -
 .../Config/start-test-grid2.xml                 |   46 -
 .../Config/start-test-grid3.xml                 |   44 -
 .../DataStructures/AtomicLongTest.cs            |    2 +-
 .../DataStructures/AtomicReferenceTest.cs       |    2 +-
 .../DataStructures/AtomicSequenceTest.cs        |    2 +-
 .../Examples/ExamplesTest.cs                    |    5 +-
 .../Apache.Ignite.Core.Tests/ExecutableTest.cs  |   49 +-
 .../IgniteConfigurationSerializerTest.cs        |   11 +-
 .../IgniteConfigurationTest.cs                  |    6 +-
 .../IgniteStartStopTest.cs                      |   79 +-
 .../Apache.Ignite.Core.Tests/IgniteTestBase.cs  |  163 -
 .../Apache.Ignite.Core.Tests/LoadDllTest.cs     |   88 +-
 .../Properties/AssemblyInfo.cs                  |    8 +-
 .../Services/ServiceProxyTest.cs                |   15 +-
 .../Services/ServicesAsyncWrapper.cs            |   18 +-
 .../Services/ServicesTest.cs                    |  148 +-
 .../Apache.Ignite.Core.Tests/SpringTestBase.cs  |  164 +
 .../Ssl/SslConfigurationTest.cs                 |  257 ++
 .../dotnet/Apache.Ignite.Core.Tests/TestBase.cs |   61 +
 .../Apache.Ignite.Core.Tests/TestRunner.cs      |   89 +-
 .../TestUtils.Common.cs                         |   40 +
 .../TestUtils.Windows.cs                        |    3 +-
 .../WindowsServiceTest.cs                       |   58 +-
 .../Apache.Ignite.Core.DotNetCore.csproj        |   15 +-
 .../Apache.Ignite.Core.Schema.nuspec            |    2 +-
 .../Apache.Ignite.Core.csproj                   |   69 +-
 .../Apache.Ignite.Core.nuspec                   |    2 +-
 .../Cache/Configuration/CacheConfiguration.cs   |    2 +
 .../Cache/Configuration/QueryEntity.cs          |    2 +-
 .../Cache/Configuration/QueryField.cs           |    2 +-
 .../Configuration/QuerySqlFieldAttribute.cs     |    2 +-
 .../Apache.Ignite.Core/Cache/ICacheMetrics.cs   |    9 +
 .../Client/Cache/CacheClientConfiguration.cs    |    2 +-
 .../Client/ISslStreamFactory.cs                 |   38 +
 .../Client/IgniteClientConfiguration.cs         |    8 +
 .../Client/SslStreamFactory.cs                  |  110 +
 .../Apache.Ignite.Core/Cluster/IBaselineNode.cs |   37 +
 .../Apache.Ignite.Core/Cluster/ICluster.cs      |   68 +-
 .../Apache.Ignite.Core/Cluster/IClusterNode.cs  |    3 +-
 .../Apache.Ignite.Core/Compute/IComputeTask.cs  |    2 +
 .../ClientConnectorConfiguration.cs             |   47 +-
 .../Configuration/DataStorageConfiguration.cs   |    3 +-
 .../Apache.Ignite.Core/Configuration/WalMode.cs |   13 +-
 .../Apache.Ignite.Core/IDataRegionMetrics.cs    |   45 +
 .../dotnet/Apache.Ignite.Core/IIgnite.cs        |   10 +
 .../IgniteClientConfigurationSection.xsd        |   34 +
 .../Apache.Ignite.Core/IgniteConfiguration.cs   |   20 +-
 .../IgniteConfigurationSection.xsd              |   69 +-
 .../dotnet/Apache.Ignite.Core/Ignition.cs       |    2 +-
 .../Impl/Binary/BinaryReader.cs                 |    8 +-
 .../Impl/Binary/BinarySystemHandlers.cs         |   16 +
 .../Impl/Binary/BinaryTypeId.cs                 |    7 +-
 .../Impl/Binary/Io/BinaryHeapStream.cs          |    6 +
 .../Impl/Binary/Io/IBinaryStream.cs             |    7 +-
 .../Impl/Binary/Metadata/BinaryType.cs          |   10 +-
 .../Impl/Binary/OptimizedMarshallerObject.cs    |   54 +
 .../Impl/Cache/CacheMetricsImpl.cs              |    7 +
 .../Query/Continuous/ContinuousQueryFilter.cs   |    2 +
 .../Cache/ClientCacheConfigurationSerializer.cs |    2 +-
 .../Impl/Client/ClientSocket.cs                 |   70 +-
 .../Impl/Cluster/BaselineNode.cs                |   89 +
 .../Impl/Cluster/ClusterNodeImpl.cs             |   38 +-
 .../Impl/Common/DelegateTypeDescriptor.cs       |    1 +
 .../Impl/Common/IgniteArgumentCheck.cs          |    1 +
 .../Impl/Common/IgniteHome.cs                   |   29 +-
 .../Impl/DataRegionMetrics.cs                   |   36 +
 .../Impl/Handle/HandleRegistry.cs               |    1 +
 .../dotnet/Apache.Ignite.Core/Impl/Ignite.cs    |   76 +-
 .../Impl/Memory/PlatformMemoryStream.cs         |   12 +-
 .../Impl/Services/DynamicServiceProxy.cs        |  108 +
 .../Impl/Services/ServiceProxyInvoker.cs        |   14 +-
 .../Impl/Services/ServiceProxySerializer.cs     |   37 +-
 .../Impl/Services/Services.cs                   |   43 +-
 .../Impl/Ssl/SslFactorySerializer.cs            |   65 +
 .../Impl/Unmanaged/Jni/Env.cs                   |    5 +-
 .../Impl/Unmanaged/Jni/Jvm.cs                   |   94 +-
 .../Impl/Unmanaged/Jni/JvmDll.cs                |   33 +-
 .../Properties/AssemblyInfo.cs                  |    8 +-
 .../Apache.Ignite.Core/Services/IServices.cs    |   26 +
 .../Ssl/ISslContextFactory.cs                   |   32 +
 .../Apache.Ignite.Core/Ssl/SslContextFactory.cs |  170 +
 .../Apache.Ignite.EntityFramework.Tests.csproj  |   22 +-
 .../Properties/AssemblyInfo.cs                  |    6 +-
 .../Apache.Ignite.EntityFramework.nuspec        |    2 +-
 .../Properties/AssemblyInfo.cs                  |    8 +-
 .../Apache.Ignite.Linq.csproj                   |    4 +
 .../Apache.Ignite.Linq.nuspec                   |    2 +-
 .../Apache.Ignite.Linq/CacheLinqExtensions.cs   |   28 +
 .../Apache.Ignite.Linq/IUpdateDescriptor.cs     |   51 +
 .../Apache.Ignite.Linq/Impl/AliasDictionary.cs  |    8 +-
 .../Impl/CacheFieldsQueryProvider.cs            |   23 +-
 .../Impl/CacheQueryExpressionVisitor.cs         |   43 +-
 .../Impl/CacheQueryModelVisitor.cs              |  147 +-
 .../Apache.Ignite.Linq/Impl/CacheQueryParser.cs |    3 +
 .../Impl/Dml/MemberUpdateContainer.cs           |   38 +
 .../Impl/Dml/UpdateAllExpressionNode.cs         |  138 +
 .../Impl/Dml/UpdateAllResultOperator.cs         |   75 +
 .../Apache.Ignite.Linq/Impl/ExpressionWalker.cs |    1 +
 .../Properties/AssemblyInfo.cs                  |    8 +-
 .../Apache.Ignite.Log4Net.nuspec                |    2 +-
 .../Properties/AssemblyInfo.cs                  |    8 +-
 .../Apache.Ignite.NLog.nuspec                   |    2 +-
 .../Properties/AssemblyInfo.cs                  |    8 +-
 modules/platforms/dotnet/Apache.Ignite.sln      |    6 +-
 .../dotnet/Apache.Ignite/IgniteRunner.cs        |    2 +-
 .../Apache.Ignite/Properties/AssemblyInfo.cs    |    8 +-
 .../Apache.Ignite/Service/IgniteService.cs      |   13 +-
 modules/platforms/dotnet/DEVNOTES.txt           |   42 +-
 modules/platforms/dotnet/README.md              |    6 +-
 modules/platforms/dotnet/build-mono.sh          |   34 +
 modules/platforms/dotnet/build.ps1              |    8 +-
 modules/platforms/dotnet/build.sh               |    7 +-
 .../dotnet/docfx/Apache.Ignite.docfx.json       |    2 +-
 .../Apache.Ignite.Examples.csproj               |    2 +
 .../Datagrid/QueryExample.cs                    |    2 +-
 .../Properties/AssemblyInfo.cs                  |    2 +-
 .../Apache.Ignite.Examples/Sql/LinqExample.cs   |    5 +-
 .../Apache.Ignite.Examples/Sql/SqlDdlExample.cs |  112 +
 .../Apache.Ignite.Examples/Sql/SqlDmlExample.cs |    5 +-
 .../Apache.Ignite.Examples/Sql/SqlExample.cs    |    5 +-
 .../ThinClient/ThinClientSqlExample.cs          |  222 +
 .../Properties/AssemblyInfo.cs                  |    2 +-
 modules/rest-http/pom.xml                       |    2 +-
 .../http/jetty/GridJettyObjectMapper.java       |   61 +-
 .../http/jetty/GridJettyRestHandler.java        |  243 +-
 modules/rocketmq/pom.xml                        |    2 +-
 modules/scalar-2.10/pom.xml                     |    4 +-
 modules/scalar/pom.xml                          |    6 +-
 modules/schedule/pom.xml                        |    2 +-
 modules/slf4j/pom.xml                           |   24 +-
 .../apache/ignite/logger/slf4j/Slf4jLogger.java |   44 +-
 .../logger/slf4j/Slf4jLoggerMarkerTest.java     |  108 +
 .../ignite/testsuites/IgniteSlf4jTestSuite.java |   37 +
 .../slf4j/src/test/resources/log4j2-test.xml    |   38 +
 modules/spark-2.10/pom.xml                      |  155 +-
 modules/spark/pom.xml                           |  187 +-
 .../ignite/spark/IgniteDataFrameSettings.scala  |  100 +
 .../spark/impl/IgniteRelationProvider.scala     |  175 +-
 .../ignite/spark/impl/IgniteSQLRelation.scala   |   93 +-
 .../apache/ignite/spark/impl/QueryHelper.scala  |  186 +
 .../apache/ignite/spark/impl/QueryUtils.scala   |  225 ++
 .../org/apache/ignite/spark/impl/package.scala  |    2 +-
 .../sql/ignite/IgniteExternalCatalog.scala      |   61 +-
 .../src/test/resources/cities_non_unique.json   |    6 +
 .../ignite/spark/AbstractDataFrameSpec.scala    |   43 +-
 .../apache/ignite/spark/IgniteCatalogSpec.scala |   14 +-
 .../ignite/spark/IgniteDataFrameSuite.scala     |    2 +
 .../spark/IgniteDataFrameWrongConfigSpec.scala  |    4 +-
 .../org/apache/ignite/spark/IgniteRDDSpec.scala |    4 +-
 ...niteSQLDataFrameIgniteSessionWriteSpec.scala |  106 +
 .../spark/IgniteSQLDataFrameWriteSpec.scala     |  371 ++
 modules/spring-data/pom.xml                     |    2 +-
 modules/spring/pom.xml                          |    2 +-
 .../ignite/cache/spring/SpringCacheManager.java |   13 +-
 modules/sqlline/pom.xml                         |    6 +-
 modules/ssh/pom.xml                             |    2 +-
 modules/storm/pom.xml                           |    6 +-
 modules/tools/pom.xml                           |   30 +-
 modules/twitter/pom.xml                         |    2 +-
 modules/urideploy/pom.xml                       |    2 +-
 modules/visor-console-2.10/pom.xml              |    2 +-
 modules/visor-console/pom.xml                   |    4 +-
 .../ignite/visor/commands/VisorConsole.scala    |   20 +-
 .../commands/cache/VisorCacheCommand.scala      |   33 +-
 .../commands/cache/VisorCacheScanCommand.scala  |   19 +-
 .../visor/commands/common/VisorTextTable.scala  |   22 +-
 .../config/VisorConfigurationCommand.scala      |    8 +
 .../visor/commands/node/VisorNodeCommand.scala  |  123 +-
 .../scala/org/apache/ignite/visor/visor.scala   |    2 +-
 modules/visor-plugins/pom.xml                   |    2 +-
 modules/web-console/.dockerignore               |    4 +
 modules/web-console/DEVNOTES.txt                |   33 +
 modules/web-console/backend/app/mongo.js        | 1179 +-----
 modules/web-console/backend/app/schemas.js      | 1175 ++++++
 modules/web-console/backend/app/settings.js     |    2 -
 modules/web-console/backend/index.js            |    4 +-
 modules/web-console/backend/package.json        |    7 +-
 modules/web-console/backend/routes/admin.js     |    2 +-
 modules/web-console/backend/routes/public.js    |    3 +-
 modules/web-console/backend/services/mails.js   |   74 +-
 .../backend/services/notifications.js           |    9 +-
 modules/web-console/backend/services/users.js   |    6 +-
 .../docker/standalone/docker-compose.yml        |   36 -
 modules/web-console/e2e/docker-compose.yml      |   41 +
 modules/web-console/e2e/testcafe/Dockerfile     |   34 +
 modules/web-console/e2e/testcafe/envtools.js    |  192 +
 .../e2e/testcafe/fixtures/admin-panel.js        |   64 +
 .../web-console/e2e/testcafe/fixtures/auth.js   |  186 +
 .../e2e/testcafe/fixtures/menu-smoke.js         |   53 +
 .../fixtures/user-profile/credentials.js        |   64 +
 .../testcafe/fixtures/user-profile/profile.js   |   84 +
 modules/web-console/e2e/testcafe/helpers.js     |   28 +
 modules/web-console/e2e/testcafe/package.json   |   47 +
 .../e2e/testcafe/page-models/PageSignIn.js      |   37 +
 modules/web-console/e2e/testcafe/roles.js       |   33 +
 modules/web-console/e2e/testcafe/testcafe.js    |   86 +
 modules/web-console/e2e/testenv/Dockerfile      |   86 +
 modules/web-console/e2e/testenv/entrypoint.sh   |   21 +
 .../web-console/e2e/testenv/nginx/nginx.conf    |   55 +
 .../e2e/testenv/nginx/web-console.conf          |   62 +
 modules/web-console/frontend/app/app.js         |   27 +-
 .../components/cluster-selector/controller.js   |    4 +-
 .../app/components/grid-export/component.js     |   20 +-
 .../app/components/grid-no-data/controller.js   |    3 +
 .../input-dialog/input-dialog.service.js        |    3 +-
 .../list-editable-transclude/directive.js       |  119 +
 .../list-editable-transclude/index.js           |   23 +
 .../app/components/list-editable/controller.js  |    1 +
 .../app/components/list-editable/index.js       |    4 +-
 .../app/components/list-editable/template.pug   |    6 +-
 .../list-of-registered-users/categories.js      |   30 +
 .../list-of-registered-users/column-defs.js     |   83 +
 .../list-of-registered-users/controller.js      |  384 ++
 .../list-of-registered-users/index.js           |   18 +-
 .../list-of-registered-users.categories.js      |   30 -
 .../list-of-registered-users.column-defs.js     |   83 -
 .../list-of-registered-users.controller.js      |  386 --
 .../list-of-registered-users.scss               |   35 -
 .../list-of-registered-users.tpl.pug            |   66 -
 .../list-of-registered-users/style.scss         |   34 +
 .../list-of-registered-users/template.tpl.pug   |   65 +
 .../frontend/app/components/page-admin/index.js |   39 +
 .../app/components/page-admin/style.scss        |   62 +
 .../app/components/page-admin/template.tpl.pug  |   28 +
 .../app/components/page-configure/index.js      |    3 +
 .../app/components/page-landing/index.js        |   65 +
 .../app/components/page-landing/style.scss      |  104 +
 .../app/components/page-landing/template.pug    |   66 +
 .../page-password-changed/controller.js         |   26 +
 .../components/page-password-changed/index.js   |   41 +
 .../components/page-password-changed/style.scss |   42 +
 .../page-password-changed/template.pug          |   22 +
 .../page-password-reset/controller.js           |   53 +
 .../app/components/page-password-reset/index.js |   57 +
 .../components/page-password-reset/style.scss   |   31 +
 .../components/page-password-reset/template.pug |   80 +
 .../app/components/page-profile/component.js    |   24 +
 .../app/components/page-profile/controller.js   |   77 +
 .../app/components/page-profile/index.js        |   47 +
 .../app/components/page-profile/style.scss      |   31 +
 .../app/components/page-profile/template.pug    |  156 +
 .../app/components/page-queries/controller.js   |   14 +-
 .../components/page-queries/template.tpl.pug    |    2 +-
 .../app/components/page-signin/controller.js    |   30 +
 .../app/components/page-signin/index.js         |   56 +
 .../app/components/page-signin/style.scss       |   38 +
 .../app/components/page-signin/template.pug     |  161 +
 .../app/components/ui-grid-filters/directive.js |   39 +-
 .../app/components/ui-grid-filters/style.scss   |   12 +
 .../app/components/ui-grid-filters/template.pug |   13 +-
 .../components/user-notifications/controller.js |    9 +-
 .../components/user-notifications/service.js    |   12 +-
 .../components/user-notifications/style.scss    |   20 +-
 .../user-notifications/template.tpl.pug         |    5 +
 .../components/web-console-header/style.scss    |    1 +
 .../components/web-console-header/template.pug  |    4 +-
 .../frontend/app/controllers/auth.controller.js |   29 -
 .../frontend/app/data/countries.json            |   74 +-
 .../app/directives/on-focus-out.directive.js    |    8 +-
 .../ui-ace-java/ui-ace-java.controller.js       |    2 +-
 .../ui-ace-spring/ui-ace-spring.controller.js   |    2 +-
 .../frontend/app/errors/CancellationError.js    |   28 +
 .../frontend/app/helpers/jade/mixins.pug        |    1 +
 .../app/modules/agent/AgentManager.service.js   |    3 +
 .../app/modules/branding/branding.service.js    |    2 +-
 .../generator/AbstractTransformer.js            |    8 +-
 .../generator/ConfigurationGenerator.js         |   73 +-
 .../generator/defaults/Cluster.service.js       |   30 +-
 .../frontend/app/modules/states/admin.state.js  |   15 +-
 .../app/modules/states/configuration.state.js   |    1 +
 .../states/configuration/caches/query.pug       |    2 +-
 .../configuration/clusters/checkpoint/s3.pug    |   28 +-
 .../configuration/clusters/client-connector.pug |   33 +-
 .../configuration/clusters/communication.pug    |    2 +-
 .../configuration/clusters/data-storage.pug     |    9 +-
 .../clusters/general/discovery/s3.pug           |   10 +-
 .../app/modules/states/password.state.js        |   50 -
 .../app/modules/states/profile.state.js         |   36 -
 .../frontend/app/modules/states/signin.state.js |   53 -
 .../frontend/app/modules/user/Auth.service.js   |    6 +-
 .../app/primitives/form-field/checkbox.pug      |   30 +
 .../app/primitives/form-field/dropdown.pug      |   51 +
 .../app/primitives/form-field/email.pug         |   37 +
 .../app/primitives/form-field/error.pug         |   29 +
 .../app/primitives/form-field/index.pug         |   27 +
 .../app/primitives/form-field/index.scss        |  149 +-
 .../app/primitives/form-field/input.pug         |   28 +
 .../app/primitives/form-field/label.pug         |   30 +
 .../app/primitives/form-field/number.pug        |   46 +
 .../app/primitives/form-field/password.pug      |   37 +
 .../app/primitives/form-field/phone.pug         |   35 +
 .../frontend/app/primitives/form-field/text.pug |   35 +
 .../app/primitives/form-field/tooltip.pug       |   18 +
 .../frontend/app/primitives/panel/index.scss    |   96 +-
 .../frontend/app/primitives/tooltip/index.scss  |   17 +
 .../frontend/app/primitives/ui-grid/index.scss  |   38 +-
 .../frontend/app/services/Confirm.service.js    |    3 +-
 .../app/services/ConfirmBatch.service.js        |    3 +-
 .../frontend/app/services/Countries.service.js  |    4 +-
 .../frontend/app/services/DefaultState.js       |   36 +
 .../frontend/app/services/Messages.service.js   |    4 +-
 .../frontend/app/services/Version.service.js    |    6 +-
 .../frontend/app/services/exceptionHandler.js   |   27 +
 .../web-console/frontend/app/services/index.js  |    2 +
 modules/web-console/frontend/app/vendor.js      |    5 +-
 .../frontend/controllers/clusters-controller.js |   14 +-
 .../frontend/controllers/profile-controller.js  |   94 -
 modules/web-console/frontend/package.json       |    9 +-
 .../frontend/public/images/icons/check.svg      |    3 +
 .../frontend/public/images/icons/collapse.svg   |    3 +
 .../public/images/icons/exclamation.svg         |    3 +
 .../frontend/public/images/icons/expand.svg     |    3 +
 .../frontend/public/images/icons/index.js       |    5 +
 .../frontend/public/images/main-screenshot.png  |  Bin 0 -> 86830 bytes
 .../public/images/page-landing-carousel-1.png   |  Bin 0 -> 84434 bytes
 .../public/images/page-landing-carousel-2.png   |  Bin 0 -> 106670 bytes
 .../public/images/page-landing-carousel-3.png   |  Bin 0 -> 70876 bytes
 .../public/stylesheets/_bootstrap-custom.scss   |    1 -
 .../stylesheets/_bootstrap-variables.scss       |    2 +-
 .../frontend/public/stylesheets/style.scss      |   90 +-
 .../web-console/frontend/test/ci/.dockerignore  |    4 +
 modules/web-console/frontend/test/ci/Dockerfile |   36 +
 .../frontend/test/ci/docker-compose.yml         |   25 +
 .../frontend/test/e2e/exampe.test.js            |   42 -
 .../frontend/test/karma.conf.babel.js           |   12 +-
 .../frontend/test/protractor.conf.js            |   50 -
 .../web-console/frontend/views/reset.tpl.pug    |   44 -
 .../frontend/views/settings/admin.tpl.pug       |   28 -
 .../frontend/views/settings/profile.tpl.pug     |   83 -
 .../web-console/frontend/views/signin.tpl.pug   |  165 -
 .../views/templates/validation-error.tpl.pug    |    2 +-
 modules/web-console/pom.xml                     |    5 +-
 modules/web-console/web-agent/pom.xml           |    2 +-
 .../ignite/console/demo/AgentClusterDemo.java   |   89 +-
 modules/web/ignite-appserver-test/pom.xml       |    2 +-
 modules/web/ignite-websphere-test/pom.xml       |    2 +-
 modules/web/pom.xml                             |    2 +-
 modules/yardstick/DEVNOTES-standalone.txt       |   10 -
 modules/yardstick/DEVNOTES.txt                  |   11 -
 .../config/benchmark-atomic.properties          |    2 +-
 ...benchmark-jdbc-thin-insert-delete.properties |   87 +
 .../benchmark-jdbc-thin-select.properties       |   86 +
 .../benchmark-jdbc-thin-update.properties       |   86 +
 .../benchmark-jdbcv2-insert-delete.properties   |   87 +
 .../config/benchmark-jdbcv2-select.properties   |   86 +
 .../config/benchmark-jdbcv2-update.properties   |   88 +
 ...enchmark-native-sql-insert-delete.properties |   87 +
 .../benchmark-native-sql-select.properties      |   86 +
 .../benchmark-native-sql-update.properties      |   86 +
 modules/yardstick/config/ignite-jdbc-config.xml |    4 +-
 .../ignite-localhost-persistence-config.xml     |   71 +
 .../benchmark-jdbc-thin-inmemory.properties     |  108 +
 .../benchmark-jdbc-thin-persistence.properties  |  113 +
 modules/yardstick/pom-standalone.xml            |   56 -
 modules/yardstick/pom.xml                       |   80 +-
 .../yardstick/IgniteBenchmarkArguments.java     |   35 +-
 .../org/apache/ignite/yardstick/IgniteNode.java |   12 +-
 .../yardstick/cache/IntMaxValueEntriesTest.java |    4 +-
 .../yardstick/jdbc/AbstractJdbcBenchmark.java   |  171 +
 .../yardstick/jdbc/AbstractNativeBenchmark.java |   37 +
 .../yardstick/jdbc/DisjointRangeGenerator.java  |  101 +
 .../jdbc/JdbcSqlInsertDeleteBenchmark.java      |   66 +
 .../jdbc/JdbcSqlQueryRangeBenchmark.java        |   78 +
 .../yardstick/jdbc/JdbcSqlUpdateBenchmark.java  |   81 +
 .../apache/ignite/yardstick/jdbc/JdbcUtils.java |   55 +
 .../jdbc/NativeSqlInsertDeleteBenchmark.java    |   61 +
 .../jdbc/NativeSqlQueryRangeBenchmark.java      |   77 +
 .../jdbc/NativeSqlUpdateRangeBenchmark.java     |   83 +
 .../apache/ignite/yardstick/ml/DataChanger.java |   65 +
 ...uzzyCMeansDistributedClustererBenchmark.java |  130 +
 ...gniteFuzzyCMeansLocalClustererBenchmark.java |   93 +
 ...niteKMeansDistributedClustererBenchmark.java |   75 +
 .../IgniteKMeansLocalClustererBenchmark.java    |   50 +
 .../yardstick/ml/clustering/package-info.java   |   22 +
 .../IgniteCholeskyDecompositionBenchmark.java   |   69 +
 .../IgniteEigenDecompositionBenchmark.java      |   69 +
 .../IgniteLUDecompositionBenchmark.java         |   75 +
 ...niteSingularValueDecompositionBenchmark.java |   59 +
 .../ml/decomposition/package-info.java          |   22 +
 .../ignite/yardstick/ml/knn/Datasets.java       |  453 +++
 .../knn/IgniteKNNClassificationBenchmark.java   |   73 +
 .../ml/knn/IgniteKNNRegressionBenchmark.java    |   82 +
 .../ignite/yardstick/ml/knn/package-info.java   |   22 +
 .../math/IgniteAbstractMatrixMulBenchmark.java  |  120 +
 ...niteDenseLocalOffHeapMatrixMulBenchmark.java |   32 +
 ...gniteDenseLocalOnHeapMatrixMulBenchmark.java |   32 +
 ...parseBlockDistributedMatrixMulBenchmark.java |   32 +
 ...iteSparseDistributedMatrixMul2Benchmark.java |   74 +
 ...niteSparseDistributedMatrixMulBenchmark.java |   49 +
 .../IgniteSparseLocalMatrixMulBenchmark.java    |   32 +
 .../ignite/yardstick/ml/math/package-info.java  |   22 +
 .../ignite/yardstick/ml/package-info.java       |   22 +
 ...iteOLSMultipleLinearRegressionBenchmark.java |   69 +
 .../yardstick/ml/regression/package-info.java   |   22 +
 .../IgniteColumnDecisionTreeGiniBenchmark.java  |   70 +
 ...niteColumnDecisionTreeVarianceBenchmark.java |   71 +
 .../yardstick/ml/trees/SplitDataGenerator.java  |  426 ++
 .../ignite/yardstick/ml/trees/package-info.java |   22 +
 .../upload/AbstractNativeBenchmark.java         |  114 +
 .../upload/AbstractUploadBenchmark.java         |  187 +
 .../upload/BatchedInsertBenchmark.java          |   65 +
 .../ignite/yardstick/upload/CopyBenchmark.java  |  125 +
 .../yardstick/upload/InsertBenchmark.java       |   52 +
 .../yardstick/upload/NativePutBenchmark.java    |   39 +
 .../upload/NativeStreamerBenchmark.java         |   66 +
 .../upload/UploadBenchmarkArguments.java        |  158 +
 .../yardstick/upload/model/QueryFactory.java    |  181 +
 .../ignite/yardstick/upload/model/Values10.java |   76 +
 .../org/yardstickframework/package-info.java    |   25 +
 .../probes/TotalTimeProbe.java                  |  137 +
 .../apache/ignite/yardstick/ml/DataChanger.java |   65 -
 ...uzzyCMeansDistributedClustererBenchmark.java |  130 -
 ...gniteFuzzyCMeansLocalClustererBenchmark.java |   93 -
 ...niteKMeansDistributedClustererBenchmark.java |   75 -
 .../IgniteKMeansLocalClustererBenchmark.java    |   50 -
 .../yardstick/ml/clustering/package-info.java   |   22 -
 .../IgniteCholeskyDecompositionBenchmark.java   |   69 -
 .../IgniteEigenDecompositionBenchmark.java      |   69 -
 .../IgniteLUDecompositionBenchmark.java         |   75 -
 ...niteSingularValueDecompositionBenchmark.java |   59 -
 .../ml/decomposition/package-info.java          |   22 -
 .../ignite/yardstick/ml/knn/Datasets.java       |  453 ---
 .../knn/IgniteKNNClassificationBenchmark.java   |   73 -
 .../ml/knn/IgniteKNNRegressionBenchmark.java    |   82 -
 .../ignite/yardstick/ml/knn/package-info.java   |   22 -
 .../math/IgniteAbstractMatrixMulBenchmark.java  |  120 -
 ...niteDenseLocalOffHeapMatrixMulBenchmark.java |   32 -
 ...gniteDenseLocalOnHeapMatrixMulBenchmark.java |   32 -
 ...parseBlockDistributedMatrixMulBenchmark.java |   32 -
 ...iteSparseDistributedMatrixMul2Benchmark.java |   74 -
 ...niteSparseDistributedMatrixMulBenchmark.java |   49 -
 .../IgniteSparseLocalMatrixMulBenchmark.java    |   32 -
 .../ignite/yardstick/ml/math/package-info.java  |   22 -
 .../ignite/yardstick/ml/package-info.java       |   22 -
 ...iteOLSMultipleLinearRegressionBenchmark.java |   69 -
 .../yardstick/ml/regression/package-info.java   |   22 -
 .../IgniteColumnDecisionTreeGiniBenchmark.java  |   70 -
 ...niteColumnDecisionTreeVarianceBenchmark.java |   71 -
 .../yardstick/ml/trees/SplitDataGenerator.java  |  426 --
 .../ignite/yardstick/ml/trees/package-info.java |   22 -
 modules/yarn/pom.xml                            |   13 +-
 modules/zeromq/pom.xml                          |    2 +-
 modules/zookeeper/pom.xml                       |    2 +-
 packaging/rpm/SPECS/apache-ignite.spec          |    8 +
 parent/pom.xml                                  |  103 +-
 pom.xml                                         |   16 +-
 scripts/update-versions.sh                      |   35 +
 2037 files changed, 111471 insertions(+), 37337 deletions(-)
----------------------------------------------------------------------



[24/50] [abbrv] ignite git commit: IGNITE-7459 Web Console: Hiding previous scan results on new scan.

Posted by sb...@apache.org.
IGNITE-7459 Web Console: Hiding previous scan results on new scan.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ae2bf3d6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ae2bf3d6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ae2bf3d6

Branch: refs/heads/ignite-zk
Commit: ae2bf3d6fc527e0f85d246abc65547d79226bf19
Parents: 536e825
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Wed Mar 14 11:33:52 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Mar 14 11:33:52 2018 +0700

----------------------------------------------------------------------
 .../frontend/app/components/page-queries/controller.js            | 3 +--
 .../frontend/app/components/page-queries/template.tpl.pug         | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/ae2bf3d6/modules/web-console/frontend/app/components/page-queries/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/controller.js b/modules/web-console/frontend/app/components/page-queries/controller.js
index 2814966..94d62b1 100644
--- a/modules/web-console/frontend/app/components/page-queries/controller.js
+++ b/modules/web-console/frontend/app/components/page-queries/controller.js
@@ -1570,10 +1570,9 @@ export default class {
             const filter = paragraph.filter;
             const pageSize = paragraph.pageSize;
 
-            paragraph.localQueryMode = local;
-
             $scope.scanAvailable(paragraph) && _chooseNode(cacheName, local)
                 .then((nid) => {
+                    paragraph.localQueryMode = local;
                     paragraph.scanningInProgress = true;
 
                     Notebook.save($scope.notebook)

http://git-wip-us.apache.org/repos/asf/ignite/blob/ae2bf3d6/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
index 4419d5c..05bd09b 100644
--- a/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
@@ -277,7 +277,7 @@ mixin paragraph-scan
                     i.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.checkScanInProgress(true)')
                     span.tipLabelExecute Scan on selected node
 
-        .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
+        .col-sm-12.sql-result(ng-if='paragraph.queryExecuted() && !paragraph.scanningInProgress' ng-switch='paragraph.resultType()')
             .error(ng-switch-when='error') Error: {{paragraph.error.message}}
             .empty(ng-switch-when='empty') Result set is empty. Duration: #[b {{paragraph.duration | duration}}]
             .table(ng-switch-when='table')


[02/50] [abbrv] ignite git commit: IGNITE-6113 Return back old implementation instead of 'removeIf' - Fixes #3609.

Posted by sb...@apache.org.
IGNITE-6113 Return back old implementation instead of 'removeIf' - Fixes #3609.

Signed-off-by: Alexey Goncharuk <al...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8c3731e8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8c3731e8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8c3731e8

Branch: refs/heads/ignite-zk
Commit: 8c3731e83296e1750c7c098b283c3fd8584b1be4
Parents: 64d920b
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Tue Mar 6 21:22:40 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Mar 6 21:22:40 2018 +0300

----------------------------------------------------------------------
 .../cache/distributed/dht/GridDhtLocalPartition.java          | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/8c3731e8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 119ea91..7a47f31 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -1194,7 +1194,12 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
     void onCacheStopped(int cacheId) {
         assert grp.sharedGroup() : grp.cacheOrGroupName();
 
-        rmvQueue.removeIf(e -> e.cacheId() == cacheId);
+        for (Iterator<RemovedEntryHolder> it = rmvQueue.iterator(); it.hasNext();) {
+            RemovedEntryHolder e = it.next();
+
+            if (e.cacheId() == cacheId)
+                it.remove();
+        }
 
         cacheMaps.remove(cacheId);
     }


[45/50] [abbrv] ignite git commit: IGNITE-7932: Add SVM example

Posted by sb...@apache.org.
IGNITE-7932: Add SVM example

this closes #3627


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f4ac81a4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f4ac81a4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f4ac81a4

Branch: refs/heads/ignite-zk
Commit: f4ac81a4881d6d4232d6749e0744d987dd2dd3d4
Parents: 094f93b
Author: zaleslaw <za...@gmail.com>
Authored: Fri Mar 16 16:34:42 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Fri Mar 16 16:34:42 2018 +0300

----------------------------------------------------------------------
 .../binary/SVMBinaryClassificationExample.java  | 233 +++++++++++++++++++
 .../examples/ml/svm/binary/package-info.java    |  22 ++
 .../ignite/examples/ml/svm/package-info.java    |  22 ++
 3 files changed, 277 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/f4ac81a4/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
new file mode 100644
index 0000000..52299f9
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.svm.binary;
+
+import java.util.Arrays;
+import java.util.UUID;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
+import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationTrainer;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * Run SVM binary-class classification model over distributed dataset.
+ *
+ * @see SVMLinearBinaryClassificationModel
+ */
+public class SVMBinaryClassificationExample {
+    /** Run example. */
+    public static void main(String[] args) throws InterruptedException {
+        System.out.println();
+        System.out.println(">>> SVM Binary classification model over cached dataset usage example started.");
+        // Start ignite grid.
+        try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+            System.out.println(">>> Ignite grid started.");
+
+            IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+                SVMBinaryClassificationExample.class.getSimpleName(), () -> {
+                IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
+
+                SVMLinearBinaryClassificationTrainer<Integer, double[]> trainer = new SVMLinearBinaryClassificationTrainer<>();
+
+                SVMLinearBinaryClassificationModel mdl = trainer.fit(
+                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
+                    (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+                    (k, v) -> v[0],
+                    4);
+
+                System.out.println(">>> SVM model " + mdl);
+
+                System.out.println(">>> ---------------------------------");
+                System.out.println(">>> | Prediction\t| Ground Truth\t|");
+                System.out.println(">>> ---------------------------------");
+
+                int amountOfErrors = 0;
+                int totalAmount = 0;
+
+                // Build confusion matrix. See https://en.wikipedia.org/wiki/Confusion_matrix
+                int[][] confusionMtx = {{0, 0}, {0, 0}};
+
+
+                try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+                    for (Cache.Entry<Integer, double[]> observation : observations) {
+                        double[] val = observation.getValue();
+                        double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+                        double groundTruth = val[0];
+
+                        double prediction = mdl.apply(new DenseLocalOnHeapVector(inputs));
+
+                        totalAmount++;
+                        if(groundTruth != prediction)
+                            amountOfErrors++;
+
+
+                        int idx1 = (int)prediction == -1.0 ? 0 : 1;
+                        int idx2 = (int)groundTruth == -1.0 ? 0 : 1;
+                        confusionMtx[idx1][idx2]++;
+
+                        System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
+                    }
+
+                    System.out.println(">>> ---------------------------------");
+
+                    System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
+                    System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
+                }
+
+                System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtx));
+            });
+
+            igniteThread.start();
+            igniteThread.join();
+        }
+    }
+
+    /**
+     * Fills cache with data and returns it.
+     *
+     * @param ignite Ignite instance.
+     * @return Filled Ignite Cache.
+     */
+    private static IgniteCache<Integer, double[]> getTestCache(Ignite ignite) {
+        CacheConfiguration<Integer, double[]> cacheConfiguration = new CacheConfiguration<>();
+        cacheConfiguration.setName("TEST_" + UUID.randomUUID());
+        cacheConfiguration.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+        IgniteCache<Integer, double[]> cache = ignite.createCache(cacheConfiguration);
+
+        for (int i = 0; i < data.length; i++)
+            cache.put(i, data[i]);
+
+        return cache;
+    }
+
+
+    /** The Iris dataset. */
+    private static final double[][] data = {
+        {-1, 5.1, 3.5, 1.4, 0.2},
+        {-1, 4.9, 3, 1.4, 0.2},
+        {-1, 4.7, 3.2, 1.3, 0.2},
+        {-1, 4.6, 3.1, 1.5, 0.2},
+        {-1, 5, 3.6, 1.4, 0.2},
+        {-1, 5.4, 3.9, 1.7, 0.4},
+        {-1, 4.6, 3.4, 1.4, 0.3},
+        {-1, 5, 3.4, 1.5, 0.2},
+        {-1, 4.4, 2.9, 1.4, 0.2},
+        {-1, 4.9, 3.1, 1.5, 0.1},
+        {-1, 5.4, 3.7, 1.5, 0.2},
+        {-1, 4.8, 3.4, 1.6, 0.2},
+        {-1, 4.8, 3, 1.4, 0.1},
+        {-1, 4.3, 3, 1.1, 0.1},
+        {-1, 5.8, 4, 1.2, 0.2},
+        {-1, 5.7, 4.4, 1.5, 0.4},
+        {-1, 5.4, 3.9, 1.3, 0.4},
+        {-1, 5.1, 3.5, 1.4, 0.3},
+        {-1, 5.7, 3.8, 1.7, 0.3},
+        {-1, 5.1, 3.8, 1.5, 0.3},
+        {-1, 5.4, 3.4, 1.7, 0.2},
+        {-1, 5.1, 3.7, 1.5, 0.4},
+        {-1, 4.6, 3.6, 1, 0.2},
+        {-1, 5.1, 3.3, 1.7, 0.5},
+        {-1, 4.8, 3.4, 1.9, 0.2},
+        {-1, 5, 3, 1.6, 0.2},
+        {-1, 5, 3.4, 1.6, 0.4},
+        {-1, 5.2, 3.5, 1.5, 0.2},
+        {-1, 5.2, 3.4, 1.4, 0.2},
+        {-1, 4.7, 3.2, 1.6, 0.2},
+        {-1, 4.8, 3.1, 1.6, 0.2},
+        {-1, 5.4, 3.4, 1.5, 0.4},
+        {-1, 5.2, 4.1, 1.5, 0.1},
+        {-1, 5.5, 4.2, 1.4, 0.2},
+        {-1, 4.9, 3.1, 1.5, 0.1},
+        {-1, 5, 3.2, 1.2, 0.2},
+        {-1, 5.5, 3.5, 1.3, 0.2},
+        {-1, 4.9, 3.1, 1.5, 0.1},
+        {-1, 4.4, 3, 1.3, 0.2},
+        {-1, 5.1, 3.4, 1.5, 0.2},
+        {-1, 5, 3.5, 1.3, 0.3},
+        {-1, 4.5, 2.3, 1.3, 0.3},
+        {-1, 4.4, 3.2, 1.3, 0.2},
+        {-1, 5, 3.5, 1.6, 0.6},
+        {-1, 5.1, 3.8, 1.9, 0.4},
+        {-1, 4.8, 3, 1.4, 0.3},
+        {-1, 5.1, 3.8, 1.6, 0.2},
+        {-1, 4.6, 3.2, 1.4, 0.2},
+        {-1, 5.3, 3.7, 1.5, 0.2},
+        {-1, 5, 3.3, 1.4, 0.2},
+        {1, 7, 3.2, 4.7, 1.4},
+        {1, 6.4, 3.2, 4.5, 1.5},
+        {1, 6.9, 3.1, 4.9, 1.5},
+        {1, 5.5, 2.3, 4, 1.3},
+        {1, 6.5, 2.8, 4.6, 1.5},
+        {1, 5.7, 2.8, 4.5, 1.3},
+        {1, 6.3, 3.3, 4.7, 1.6},
+        {1, 4.9, 2.4, 3.3, 1},
+        {1, 6.6, 2.9, 4.6, 1.3},
+        {1, 5.2, 2.7, 3.9, 1.4},
+        {1, 5, 2, 3.5, 1},
+        {1, 5.9, 3, 4.2, 1.5},
+        {1, 6, 2.2, 4, 1},
+        {1, 6.1, 2.9, 4.7, 1.4},
+        {1, 5.6, 2.9, 3.6, 1.3},
+        {1, 6.7, 3.1, 4.4, 1.4},
+        {1, 5.6, 3, 4.5, 1.5},
+        {1, 5.8, 2.7, 4.1, 1},
+        {1, 6.2, 2.2, 4.5, 1.5},
+        {1, 5.6, 2.5, 3.9, 1.1},
+        {1, 5.9, 3.2, 4.8, 1.8},
+        {1, 6.1, 2.8, 4, 1.3},
+        {1, 6.3, 2.5, 4.9, 1.5},
+        {1, 6.1, 2.8, 4.7, 1.2},
+        {1, 6.4, 2.9, 4.3, 1.3},
+        {1, 6.6, 3, 4.4, 1.4},
+        {1, 6.8, 2.8, 4.8, 1.4},
+        {1, 6.7, 3, 5, 1.7},
+        {1, 6, 2.9, 4.5, 1.5},
+        {1, 5.7, 2.6, 3.5, 1},
+        {1, 5.5, 2.4, 3.8, 1.1},
+        {1, 5.5, 2.4, 3.7, 1},
+        {1, 5.8, 2.7, 3.9, 1.2},
+        {1, 6, 2.7, 5.1, 1.6},
+        {1, 5.4, 3, 4.5, 1.5},
+        {1, 6, 3.4, 4.5, 1.6},
+        {1, 6.7, 3.1, 4.7, 1.5},
+        {1, 6.3, 2.3, 4.4, 1.3},
+        {1, 5.6, 3, 4.1, 1.3},
+        {1, 5.5, 2.5, 4, 1.3},
+        {1, 5.5, 2.6, 4.4, 1.2},
+        {1, 6.1, 3, 4.6, 1.4},
+        {1, 5.8, 2.6, 4, 1.2},
+        {1, 5, 2.3, 3.3, 1},
+        {1, 5.6, 2.7, 4.2, 1.3},
+        {1, 5.7, 3, 4.2, 1.2},
+        {1, 5.7, 2.9, 4.2, 1.3},
+        {1, 6.2, 2.9, 4.3, 1.3},
+        {1, 5.1, 2.5, 3, 1.1},
+        {1, 5.7, 2.8, 4.1, 1.3},
+    };
+
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/f4ac81a4/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/package-info.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/package-info.java
new file mode 100644
index 0000000..22c9ad7
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * SVM Binary Classification Examples.
+ */
+package org.apache.ignite.examples.ml.svm.binary;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/f4ac81a4/examples/src/main/java/org/apache/ignite/examples/ml/svm/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/package-info.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/package-info.java
new file mode 100644
index 0000000..a2fef75
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * SVM examples.
+ */
+package org.apache.ignite.examples.ml.svm;
\ No newline at end of file


[06/50] [abbrv] ignite git commit: IGNITE-7851: .NET: CacheConfiguration(string, Type[]) depricated.

Posted by sb...@apache.org.
IGNITE-7851: .NET: CacheConfiguration(string, Type[]) depricated.

This closes #3595


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/caba7cca
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/caba7cca
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/caba7cca

Branch: refs/heads/ignite-zk
Commit: caba7cca8603dc0f7ae7c0faba714f089448bf72
Parents: ed76af2
Author: apopov <ta...@gmail.com>
Authored: Wed Mar 7 12:58:32 2018 +0300
Committer: Igor Sapego <is...@gridgain.com>
Committed: Wed Mar 7 12:58:32 2018 +0300

----------------------------------------------------------------------
 .../Cache/Query/CacheQueriesCodeConfigurationTest.cs           | 2 +-
 .../Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs   | 6 ++++--
 .../Cache/Query/Linq/CacheLinqTest.Misc.cs                     | 4 ++--
 .../Cache/Query/Linq/CacheLinqTest.Numerics.cs                 | 4 ++--
 .../Apache.Ignite.Core.Tests/Client/Cache/CacheTestNoMeta.cs   | 2 +-
 .../Cache/Configuration/CacheConfiguration.cs                  | 2 ++
 .../examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs   | 2 +-
 .../dotnet/examples/Apache.Ignite.Examples/Sql/LinqExample.cs  | 5 +++--
 .../dotnet/examples/Apache.Ignite.Examples/Sql/SqlExample.cs   | 5 +++--
 9 files changed, 19 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
index 9768fca..a05923d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
@@ -161,7 +161,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
             using (var ignite = Ignition.Start(cfg))
             {
                 var cache = ignite.GetOrCreateCache<int, AttributeQueryPerson>(new CacheConfiguration(CacheName,
-                        typeof (AttributeQueryPerson)));
+                        new QueryEntity(typeof(int), typeof(AttributeQueryPerson))));
 
                 Assert.IsNotNull(cache);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
index 278d1fd..ceeeb37 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
@@ -616,8 +616,10 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
         [Test]
         public void TestCustomSchema()
         {
-            var doubles = GetIgnite().GetOrCreateCache<int, double>(new CacheConfiguration("doubles", typeof(double)));
-            var strings = GetIgnite().GetOrCreateCache<int, string>(new CacheConfiguration("strings", typeof(string)));
+            var doubles = GetIgnite().GetOrCreateCache<int, double>(new CacheConfiguration("doubles",
+                    new QueryEntity(typeof(int), typeof(double))));
+            var strings = GetIgnite().GetOrCreateCache<int, string>(new CacheConfiguration("strings",
+                    new QueryEntity(typeof(int), typeof(string))));
 
             doubles[1] = 36.6;
             strings[1] = "foo";

http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Misc.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Misc.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Misc.cs
index 3585281..d9c57d0 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Misc.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Misc.cs
@@ -237,8 +237,8 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
         public void TestLocalQuery()
         {
             // Create partitioned cache
-            var cache =
-                Ignition.GetIgnite().GetOrCreateCache<int, int>(new CacheConfiguration("partCache", typeof(int))
+            var cache = Ignition.GetIgnite().GetOrCreateCache<int, int>(new CacheConfiguration("partCache", 
+                    new QueryEntity(typeof(int), typeof(int)))
                 {
                     SqlEscapeAll = GetSqlEscapeAll()
                 });

http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Numerics.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Numerics.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Numerics.cs
index 3f973aa..8dc7ddf 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Numerics.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Numerics.cs
@@ -43,8 +43,8 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
         [Test]
         public void TestNumerics()
         {
-            var cache = Ignition.GetIgnite()
-                .GetOrCreateCache<int, Numerics>(new CacheConfiguration("numerics", typeof(Numerics))
+            var cache = Ignition.GetIgnite().GetOrCreateCache<int, Numerics>(new CacheConfiguration("numerics", 
+                    new QueryEntity(typeof(int), typeof(Numerics)))
                 {
                     SqlEscapeAll = GetSqlEscapeAll()
                 });

http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/CacheTestNoMeta.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/CacheTestNoMeta.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/CacheTestNoMeta.cs
index a2ca65d..0a8b146 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/CacheTestNoMeta.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/CacheTestNoMeta.cs
@@ -52,7 +52,7 @@ namespace Apache.Ignite.Core.Tests.Client.Cache
             using (var client = Ignition.StartClient(cfg))
             {
                 var serverCache = Ignition.GetIgnite().GetOrCreateCache<int?, Person>(
-                    new CacheConfiguration("person", typeof(Person)));
+                    new CacheConfiguration("person", new QueryEntity(typeof(int?), typeof(Person))));
 
                 var clientCache = client.GetCache<int?, Person>(serverCache.Name);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
index 4880eb3..06a7d72 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
@@ -218,12 +218,14 @@ namespace Apache.Ignite.Core.Cache.Configuration
         /// <summary>
         /// Initializes a new instance of the <see cref="CacheConfiguration"/> class 
         /// and populates <see cref="QueryEntities"/> according to provided query types.
+        /// This constructor is depricated, please use <see cref="CacheConfiguration(string, QueryEntity[])"/>
         /// </summary>
         /// <param name="name">Cache name.</param>
         /// <param name="queryTypes">
         /// Collection of types to be registered as query entities. These types should use 
         /// <see cref="QuerySqlFieldAttribute"/> to configure query fields and properties.
         /// </param>
+        [Obsolete("This constructor is deprecated, please use CacheConfiguration(string, QueryEntity[]) instead.")]
         public CacheConfiguration(string name, params Type[] queryTypes) : this(name)
         {
             QueryEntities = queryTypes.Select(type => new QueryEntity {ValueType = type}).ToArray();

http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs
index 82bd72a..a059b5a 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs
@@ -53,7 +53,7 @@ namespace Apache.Ignite.Examples.Datagrid
                 Console.WriteLine(">>> Cache query example started.");
 
                 var employeeCache = ignite.GetOrCreateCache<int, Employee>(
-                    new CacheConfiguration(EmployeeCacheName, typeof(Employee)));
+                    new CacheConfiguration(EmployeeCacheName, new QueryEntity(typeof(int), typeof(Employee))));
 
                 // Populate cache with sample data entries.
                 PopulateCache(employeeCache);

http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/LinqExample.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/LinqExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/LinqExample.cs
index 9962b20..ee2fd90 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/LinqExample.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/LinqExample.cs
@@ -60,10 +60,11 @@ namespace Apache.Ignite.Examples.Sql
                 Console.WriteLine(">>> Cache LINQ example started.");
 
                 var employeeCache = ignite.GetOrCreateCache<int, Employee>(
-                    new CacheConfiguration(EmployeeCacheName, typeof(Employee)));
+                    new CacheConfiguration(EmployeeCacheName, new QueryEntity(typeof(int), typeof(Employee))));
 
                 var employeeCacheColocated = ignite.GetOrCreateCache<AffinityKey, Employee>(
-                    new CacheConfiguration(EmployeeCacheNameColocated, typeof(Employee)));
+                    new CacheConfiguration(EmployeeCacheNameColocated, 
+                        new QueryEntity(typeof(AffinityKey), typeof(Employee))));
 
                 var organizationCache = ignite.GetOrCreateCache<int, Organization>(
                     new CacheConfiguration(OrganizationCacheName, new QueryEntity(typeof(int), typeof(Organization))));

http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlExample.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlExample.cs
index 7222cfb..15b9d40 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlExample.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlExample.cs
@@ -58,10 +58,11 @@ namespace Apache.Ignite.Examples.Sql
                 Console.WriteLine(">>> Cache query example started.");
 
                 var employeeCache = ignite.GetOrCreateCache<int, Employee>(
-                    new CacheConfiguration(EmployeeCacheName, typeof(Employee)));
+                    new CacheConfiguration(EmployeeCacheName, new QueryEntity(typeof(int), typeof(Employee))));
 
                 var employeeCacheColocated = ignite.GetOrCreateCache<AffinityKey, Employee>(
-                    new CacheConfiguration(EmployeeCacheNameColocated, typeof(Employee)));
+                    new CacheConfiguration(EmployeeCacheNameColocated,
+                        new QueryEntity(typeof(AffinityKey), typeof(Employee))));
 
                 var organizationCache = ignite.GetOrCreateCache<int, Organization>(
                     new CacheConfiguration(OrganizationCacheName, new QueryEntity(typeof(int), typeof(Organization))));


[10/50] [abbrv] ignite git commit: IGNITE-7869 Dynamic start cache by stored cache data

Posted by sb...@apache.org.
IGNITE-7869 Dynamic start cache by stored cache data


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4ee181ce
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4ee181ce
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4ee181ce

Branch: refs/heads/ignite-zk
Commit: 4ee181ce42972cb5197d780a79e0a96c26fe77e4
Parents: 90c0af8
Author: Anton Kalashnikov <ka...@yandex.ru>
Authored: Mon Mar 12 12:53:36 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Mar 12 12:53:36 2018 +0300

----------------------------------------------------------------------
 .../cache/DynamicCacheDescriptor.java           |  17 ++-
 .../processors/cache/GridCacheProcessor.java    | 139 +++++++++----------
 .../processors/cache/StoredCacheData.java       |   9 +-
 3 files changed, 86 insertions(+), 79 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/4ee181ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
index 4771c3e..cad8414 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
@@ -284,7 +284,6 @@ public class DynamicCacheDescriptor {
         this.rcvdFromVer = rcvdFromVer;
     }
 
-
     /**
      * @return Start topology version or {@code null} if cache configured statically.
      */
@@ -346,6 +345,22 @@ public class DynamicCacheDescriptor {
         }
     }
 
+    /**
+     * Form a {@link StoredCacheData} with all data to correctly restore cache params when its configuration is read
+     * from page store. Essentially, this method takes from {@link DynamicCacheDescriptor} all that's needed to start
+     * cache correctly, leaving out everything else.
+     */
+    public StoredCacheData toStoredData() {
+        assert schema != null;
+
+        StoredCacheData res = new StoredCacheData(cacheConfiguration());
+
+        res.queryEntities(schema().entities());
+        res.sql(sql());
+
+        return res;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(DynamicCacheDescriptor.class, this, "cacheName", U.maskName(cacheCfg.getName()));

http://git-wip-us.apache.org/repos/asf/ignite/blob/4ee181ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 4648f25..eec964a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -34,6 +34,7 @@ import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
+import java.util.stream.Collectors;
 import javax.management.MBeanServer;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -99,8 +100,8 @@ import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaS
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FsyncModeFileWriteAheadLogManager;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FsyncModeFileWriteAheadLogManager;
 import org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager;
 import org.apache.ignite.internal.processors.cache.query.GridCacheLocalQueryManager;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
@@ -508,7 +509,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
             assertParameter(cc.getTransactionManagerLookupClassName() == null,
                 "transaction manager can not be used with ATOMIC cache");
 
-        if ((cc.getEvictionPolicyFactory() != null || cc.getEvictionPolicy() != null)&& !cc.isOnheapCacheEnabled())
+        if ((cc.getEvictionPolicyFactory() != null || cc.getEvictionPolicy() != null) && !cc.isOnheapCacheEnabled())
             throw new IgniteCheckedException("Onheap cache must be enabled if eviction policy is configured [cacheName="
                 + U.maskName(cc.getName()) + "]");
 
@@ -1167,12 +1168,12 @@ public class GridCacheProcessor extends GridProcessorAdapter {
 
         if (log.isInfoEnabled()) {
             log.info("Started cache [name=" + cfg.getName() +
-                ", id="+cacheCtx.cacheId() +
+                ", id=" + cacheCtx.cacheId() +
                 (cfg.getGroupName() != null ? ", group=" + cfg.getGroupName() : "") +
                 ", memoryPolicyName=" + memPlcName +
                 ", mode=" + cfg.getCacheMode() +
                 ", atomicity=" + cfg.getAtomicityMode() +
-                ", backups=" + cfg.getBackups() +']');
+                ", backups=" + cfg.getBackups() + ']');
         }
     }
 
@@ -1189,9 +1190,10 @@ public class GridCacheProcessor extends GridProcessorAdapter {
             if (!cache.isNear() && ctx.shared().wal() != null) {
                 try {
                     ctx.shared().wal().fsync(null);
-                } catch (IgniteCheckedException e) {
+                }
+                catch (IgniteCheckedException e) {
                     U.error(log, "Failed to flush write-ahead log on cache stop " +
-                            "[cache=" + ctx.name() + "]", e);
+                        "[cache=" + ctx.name() + "]", e);
                 }
             }
 
@@ -1343,8 +1345,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
      * @param cacheObjCtx Cache object context.
      * @param affNode {@code True} if local node affinity node.
      * @param updatesAllowed Updates allowed flag.
-     * @param disabledAfterStart If true, then we will discard restarting state from proxies. If false then we will change
-     *  state of proxies to restarting
+     * @param disabledAfterStart If true, then we will discard restarting state from proxies. If false then we will
+     * change state of proxies to restarting
      * @return Cache context.
      * @throws IgniteCheckedException If failed to create cache.
      */
@@ -1845,10 +1847,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
             ccfg.setNearConfiguration(reqNearCfg);
         }
 
-        StoredCacheData cacheData = toStoredData(desc);
-
         if (sharedCtx.pageStore() != null && affNode)
-            sharedCtx.pageStore().initializeForCache(desc.groupDescriptor(), cacheData);
+            sharedCtx.pageStore().initializeForCache(desc.groupDescriptor(), desc.toStoredData());
 
         String grpName = startCfg.getGroupName();
 
@@ -1923,7 +1923,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
             if (proxy == null)
                 continue;
 
-             GridCacheContext<?, ?> cacheCtx = sharedCtx.cacheContext(CU.cacheId(proxy.getName()));
+            GridCacheContext<?, ?> cacheCtx = sharedCtx.cacheContext(CU.cacheId(proxy.getName()));
 
             if (cacheCtx == null)
                 continue;
@@ -2091,7 +2091,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
         for (GridCacheAdapter<?, ?> cache : caches.values()) {
             GridCacheContext<?, ?> cacheCtx = cache.context();
 
-            if (cacheCtx.startTopologyVersion().equals(startTopVer) ) {
+            if (cacheCtx.startTopologyVersion().equals(startTopVer)) {
                 if (!jCacheProxies.containsKey(cacheCtx.name())) {
                     IgniteCacheProxyImpl newProxy = new IgniteCacheProxyImpl(cache.context(), cache, false);
 
@@ -2444,8 +2444,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
      * @param msg Message.
      * @param topVer Current topology version.
      * @param curState Current cluster state.
-     * @throws IgniteCheckedException If configuration validation failed.
      * @return Exchange actions.
+     * @throws IgniteCheckedException If configuration validation failed.
      */
     public ExchangeActions onStateChangeRequest(
         ChangeGlobalStateMessage msg,
@@ -2595,8 +2595,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
                 return new GridFinishedFuture<>();
 
             CacheConfiguration ccfg = F.isEmpty(templateName)
-                ?  getOrCreateConfigFromTemplate(cacheName)
-                :  getOrCreateConfigFromTemplate(templateName);
+                ? getOrCreateConfigFromTemplate(cacheName)
+                : getOrCreateConfigFromTemplate(templateName);
 
             ccfg.setName(cacheName);
 
@@ -2785,7 +2785,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
                 sql,
                 failIfExists,
                 failIfNotStarted,
-                false);
+                false,
+                null);
 
             if (req != null) {
                 if (req.clientStartOnly())
@@ -2837,26 +2838,28 @@ public class GridCacheProcessor extends GridProcessorAdapter {
      */
     public IgniteInternalFuture<?> dynamicStartCaches(Collection<CacheConfiguration> ccfgList, boolean failIfExists,
         boolean checkThreadTx, boolean disabledAfterStart) {
-        return dynamicStartCaches(ccfgList, null, failIfExists, checkThreadTx, disabledAfterStart);
+        return dynamicStartCachesByStoredConf(
+            ccfgList.stream().map(StoredCacheData::new).collect(Collectors.toList()),
+            failIfExists,
+            checkThreadTx,
+            disabledAfterStart
+        );
     }
 
     /**
      * Dynamically starts multiple caches.
      *
-     * @param ccfgList Collection of cache configuration.
-     * @param cacheType Cache type.
+     * @param storedCacheDataList Collection of stored cache data.
      * @param failIfExists Fail if exists flag.
      * @param checkThreadTx If {@code true} checks that current thread does not have active transactions.
      * @param disabledAfterStart If true, cache proxies will be only activated after {@link #restartProxies()}.
      * @return Future that will be completed when all caches are deployed.
      */
-    private IgniteInternalFuture<?> dynamicStartCaches(
-        Collection<CacheConfiguration> ccfgList,
-        CacheType cacheType,
+    public IgniteInternalFuture<?> dynamicStartCachesByStoredConf(
+        Collection<StoredCacheData> storedCacheDataList,
         boolean failIfExists,
         boolean checkThreadTx,
-        boolean disabledAfterStart
-    ) {
+        boolean disabledAfterStart) {
         if (checkThreadTx)
             checkEmptyTransactions();
 
@@ -2864,40 +2867,28 @@ public class GridCacheProcessor extends GridProcessorAdapter {
         Map<String, DynamicCacheChangeRequest> clientReqs = null;
 
         try {
-            for (CacheConfiguration ccfg : ccfgList) {
-                CacheType ct = cacheType;
-
-                if (ct == null) {
-                    if (CU.isUtilityCache(ccfg.getName()))
-                        ct = CacheType.UTILITY;
-                    else if (internalCaches.contains(ccfg.getName()))
-                        ct = CacheType.INTERNAL;
-                    else if (DataStructuresProcessor.isDataStructureCache(ccfg.getName()))
-                        ct = CacheType.DATA_STRUCTURES;
-                    else
-                        ct = CacheType.USER;
-                }
-
+            for (StoredCacheData ccfg : storedCacheDataList) {
                 DynamicCacheChangeRequest req = prepareCacheChangeRequest(
-                    ccfg,
-                    ccfg.getName(),
+                    ccfg.config(),
+                    ccfg.config().getName(),
                     null,
-                    ct,
-                    false,
+                    resolveCacheType(ccfg.config()),
+                    ccfg.sql(),
                     failIfExists,
                     true,
-                    disabledAfterStart);
+                    disabledAfterStart,
+                    ccfg.queryEntities());
 
                 if (req != null) {
                     if (req.clientStartOnly()) {
                         if (clientReqs == null)
-                            clientReqs = U.newLinkedHashMap(ccfgList.size());
+                            clientReqs = U.newLinkedHashMap(storedCacheDataList.size());
 
                         clientReqs.put(req.cacheName(), req);
                     }
                     else {
                         if (srvReqs == null)
-                            srvReqs = new ArrayList<>(ccfgList.size());
+                            srvReqs = new ArrayList<>(storedCacheDataList.size());
 
                         srvReqs.add(req);
                     }
@@ -2931,6 +2922,18 @@ public class GridCacheProcessor extends GridProcessorAdapter {
             return new GridFinishedFuture<>();
     }
 
+    /** Resolve cache type for input cacheType */
+    @NotNull private CacheType resolveCacheType(CacheConfiguration ccfg) {
+        if (CU.isUtilityCache(ccfg.getName()))
+            return CacheType.UTILITY;
+        else if (internalCaches.contains(ccfg.getName()))
+            return CacheType.INTERNAL;
+        else if (DataStructuresProcessor.isDataStructureCache(ccfg.getName()))
+            return CacheType.DATA_STRUCTURES;
+        else
+            return CacheType.USER;
+    }
+
     /**
      * @param cacheName Cache name to destroy.
      * @param sql If the cache needs to be destroyed only if it was created as the result of SQL {@code CREATE TABLE}
@@ -2989,6 +2992,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
 
     /**
      * Prepares cache stop request.
+     *
      * @param cacheName Cache names to destroy.
      * @param restart Restart flag.
      * @param destroy Cache data destroy flag. Setting to {@code true} will cause removing all cache data from store.
@@ -3006,6 +3010,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
 
     /**
      * Starts cache stop request as cache change batch.
+     *
      * @param reqs cache stop requests.
      * @return compound future.
      */
@@ -3108,7 +3113,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
      * @param cacheName Cache name.
      * @return Cache type.
      */
-    public CacheType cacheType(String cacheName ) {
+    public CacheType cacheType(String cacheName) {
         if (CU.isUtilityCache(cacheName))
             return CacheType.UTILITY;
         else if (internalCaches.contains(cacheName))
@@ -3128,30 +3133,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
         assert desc != null;
 
         if (sharedCtx.pageStore() != null && !sharedCtx.kernalContext().clientNode() &&
-            CU.isPersistentCache(desc.cacheConfiguration(), sharedCtx.gridConfig().getDataStorageConfiguration())) {
-            StoredCacheData data = toStoredData(desc);
-
-            sharedCtx.pageStore().storeCacheData(data, true);
-        }
-    }
-
-    /**
-     * Form a {@link StoredCacheData} with all data to correctly restore cache params when its configuration
-     * is read from page store. Essentially, this method takes from {@link DynamicCacheDescriptor} all that's
-     * needed to start cache correctly, leaving out everything else.
-     *
-     * @param desc Cache descriptor to process.
-     * @return {@link StoredCacheData} based on {@code desc}.
-     */
-    private static StoredCacheData toStoredData(DynamicCacheDescriptor desc) {
-        A.notNull(desc, "desc");
-
-        StoredCacheData res = new StoredCacheData(desc.cacheConfiguration());
-
-        res.queryEntities(desc.schema() == null ? Collections.<QueryEntity>emptyList() : desc.schema().entities());
-        res.sql(desc.sql());
-
-        return res;
+            CU.isPersistentCache(desc.cacheConfiguration(), sharedCtx.gridConfig().getDataStorageConfiguration()))
+            sharedCtx.pageStore().storeCacheData(desc.toStoredData(), true);
     }
 
     /**
@@ -3355,7 +3338,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     /**
      * Reset restarting caches.
      */
-    public void resetRestartingCaches(){
+    public void resetRestartingCaches() {
         cachesInfo.restartingCaches().clear();
     }
 
@@ -3677,7 +3660,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
             cache = jCacheProxies.get(cacheName);
         }
 
-        return cache != null ? (IgniteCacheProxy<K, V>) cache.gatewayWrapper() : null;
+        return cache != null ? (IgniteCacheProxy<K, V>)cache.gatewayWrapper() : null;
     }
 
     /**
@@ -3955,7 +3938,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     @SuppressWarnings("unchecked")
     private void registerMbean(Object obj, @Nullable String cacheName, boolean near)
         throws IgniteCheckedException {
-        if(U.IGNITE_MBEANS_DISABLED)
+        if (U.IGNITE_MBEANS_DISABLED)
             return;
 
         assert obj != null;
@@ -3993,7 +3976,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
      * @param near Near flag.
      */
     private void unregisterMbean(Object o, @Nullable String cacheName, boolean near) {
-        if(U.IGNITE_MBEANS_DISABLED)
+        if (U.IGNITE_MBEANS_DISABLED)
             return;
 
         assert o != null;
@@ -4141,6 +4124,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
      * @param failIfExists Fail if exists flag.
      * @param failIfNotStarted If {@code true} fails if cache is not started.
      * @param disabledAfterStart If true, cache proxies will be only activated after {@link #restartProxies()}.
+     * @param qryEntities Query entities.
      * @return Request or {@code null} if cache already exists.
      * @throws IgniteCheckedException if some of pre-checks failed
      * @throws CacheExistsException if cache exists and failIfExists flag is {@code true}
@@ -4153,7 +4137,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
         boolean sql,
         boolean failIfExists,
         boolean failIfNotStarted,
-        boolean disabledAfterStart
+        boolean disabledAfterStart,
+        @Nullable Collection<QueryEntity> qryEntities
     ) throws IgniteCheckedException {
         DynamicCacheDescriptor desc = cacheDescriptor(cacheName);
 
@@ -4208,7 +4193,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
                 initialize(cfg, cacheObjCtx);
 
                 req.startCacheConfiguration(cfg);
-                req.schema(new QuerySchema(cfg.getQueryEntities()));
+                req.schema(new QuerySchema(qryEntities != null ? qryEntities : cfg.getQueryEntities()));
             }
         }
         else {

http://git-wip-us.apache.org/repos/asf/ignite/blob/4ee181ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java
index 5a88036..26b4f9b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java
@@ -39,7 +39,7 @@ public class StoredCacheData implements Serializable {
 
     /** Cache configuration. */
     @GridToStringInclude
-    private final CacheConfiguration<?, ?> ccfg;
+    private CacheConfiguration<?, ?> ccfg;
 
     /** Query entities. */
     @GridToStringInclude
@@ -61,6 +61,13 @@ public class StoredCacheData implements Serializable {
     }
 
     /**
+     * @param ccfg Cache configuration.
+     */
+    public void config(CacheConfiguration<?, ?> ccfg) {
+        this.ccfg = ccfg;
+    }
+
+    /**
      * @return Cache configuration.
      */
     public CacheConfiguration<?, ?> config() {


[05/50] [abbrv] ignite git commit: IGNITE-5298 .NET: DML update via LINQ

Posted by sb...@apache.org.
IGNITE-5298 .NET: DML update via LINQ

This closes #3599


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ed76af24
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ed76af24
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ed76af24

Branch: refs/heads/ignite-zk
Commit: ed76af245e7947b0e701907bde76849971a6a75c
Parents: 3108700
Author: Sergey Stronchinskiy <gu...@gmail.com>
Authored: Wed Mar 7 11:44:51 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Wed Mar 7 11:44:51 2018 +0300

----------------------------------------------------------------------
 .../Cache/Query/Linq/CacheLinqTest.Base.cs      |   4 +
 .../Cache/Query/Linq/CacheLinqTest.Custom.cs    | 195 +++++++++++++++++++
 .../Apache.Ignite.Linq.csproj                   |   4 +
 .../Apache.Ignite.Linq/CacheLinqExtensions.cs   |  28 +++
 .../Apache.Ignite.Linq/IUpdateDescriptor.cs     |  51 +++++
 .../Apache.Ignite.Linq/Impl/AliasDictionary.cs  |   8 +-
 .../Impl/CacheQueryExpressionVisitor.cs         |  43 +++-
 .../Impl/CacheQueryModelVisitor.cs              | 143 ++++++++++----
 .../Apache.Ignite.Linq/Impl/CacheQueryParser.cs |   3 +
 .../Impl/Dml/MemberUpdateContainer.cs           |  38 ++++
 .../Impl/Dml/UpdateAllExpressionNode.cs         | 138 +++++++++++++
 .../Impl/Dml/UpdateAllResultOperator.cs         |  75 +++++++
 12 files changed, 680 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Base.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Base.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Base.cs
index d70a7a4..5b56abd 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Base.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Base.cs
@@ -346,6 +346,8 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
 
             [QuerySqlField] public int AliasTest { get; set; }
 
+            [QuerySqlField] public bool Bool { get; set; }
+
             public void WriteBinary(IBinaryWriter writer)
             {
                 writer.WriteInt("age1", Age);
@@ -354,6 +356,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
                 writer.WriteObject("Address", Address);
                 writer.WriteTimestamp("Birthday", Birthday);
                 writer.WriteInt("AliasTest", AliasTest);
+                writer.WriteBoolean("Bool", Bool);
             }
 
             public void ReadBinary(IBinaryReader reader)
@@ -364,6 +367,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
                 Address = reader.ReadObject<Address>("Address");
                 Birthday = reader.ReadTimestamp("Birthday");
                 AliasTest = reader.ReadInt("AliasTest");
+                Bool = reader.ReadBoolean("Bool");
             }
         }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Custom.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Custom.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Custom.cs
index f797b4c..c854f1f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Custom.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Custom.cs
@@ -28,6 +28,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
 {
     using System;
     using System.Linq;
+    using Apache.Ignite.Core.Cache;
     using Apache.Ignite.Core.Cache.Configuration;
     using Apache.Ignite.Core.Common;
     using Apache.Ignite.Linq;
@@ -103,5 +104,199 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
             var ex = Assert.Throws<IgniteException>(() => qry.RemoveAll());
             Assert.AreEqual("Failed to parse query", ex.Message.Substring(0, 21));
         }
+
+        /// <summary>
+        /// Tests the UpdateAll extension without condition.
+        /// </summary>
+        [Test]
+        public void TestUpdateAllUnconditional()
+        {
+            // Use new cache to avoid touching static data.
+            var personCount = 10;
+            var orgCount = 3;
+            var personQueryable = GetPersonCacheQueryable("updateAllTest_Unconditional_Persons", personCount, orgCount);
+            var orgQueryable = GetOrgCacheQueryable("updateAllTest_Unconditional_Org", orgCount);
+
+            var allOrg = orgQueryable.ToArray();
+
+            // Constant value
+            var updated = personQueryable
+                .UpdateAll(d => d.Set(p => p.AliasTest, 7));
+            Assert.AreEqual(personCount, updated);
+            AssertAll(personQueryable, p => p.Value.AliasTest == 7);
+
+            // Expression value - from self
+            updated = personQueryable
+                .UpdateAll(d => d.Set(p => p.AliasTest, e => e.Key));
+            Assert.AreEqual(personCount, updated);
+            AssertAll(personQueryable, p => p.Value.AliasTest == p.Key);
+
+            // Multiple sets
+            var aliasValue = 3;
+            updated = personQueryable
+                .UpdateAll(d => d.Set(p => p.AliasTest, aliasValue).Set(p => p.Name, aliasValue.ToString()));
+            Assert.AreEqual(personCount, updated);
+            AssertAll(personQueryable, p => p.Value.AliasTest == aliasValue && p.Value.Name == aliasValue.ToString());
+
+            // Expression value - subquery with same cache
+            updated = personQueryable
+                .UpdateAll(d => d.Set(p => p.AliasTest,
+                    e => personQueryable.Where(ie => ie.Key == e.Key).Select(ie => ie.Key).First()));
+            Assert.AreEqual(personCount, updated);
+            AssertAll(personQueryable, p => p.Value.AliasTest == p.Key);
+
+            // Expression value - subquery with other cache
+            updated = personQueryable
+                .UpdateAll(d => d.Set(p => p.AliasTest, p => orgQueryable.Count(o => o.Value.Id > p.Key)));
+            Assert.AreEqual(personCount, updated);
+            AssertAll(personQueryable, p => p.Value.AliasTest == allOrg.Count(o => o.Value.Id > p.Key));
+
+            updated = personQueryable
+                .UpdateAll(d => d.Set(p => p.Name,
+                    e => orgQueryable.Where(o => o.Key == e.Value.OrganizationId).Select(o => o.Value.Name).First()));
+            Assert.AreEqual(personCount, updated);
+            AssertAll(personQueryable,
+                p => p.Value.Name == allOrg.Where(o => o.Key == p.Value.OrganizationId).Select(o => o.Value.Name)
+                         .FirstOrDefault());
+
+            // Expression value - Contains subquery with other cache
+            updated = personQueryable
+                .UpdateAll(d => d.Set(p => p.Bool,
+                    p => orgQueryable.Select(o => o.Key).Contains(p.Value.OrganizationId)));
+            Assert.AreEqual(personCount, updated);
+            AssertAll(personQueryable,
+                 p => p.Value.Bool == allOrg.Select(o => o.Key).Contains(p.Value.OrganizationId));
+
+            // Row number limit.
+            var name = "rowLimit" + 2;
+            updated = personQueryable
+                .Take(2)
+                .UpdateAll(d => d.Set(p => p.Name, name));
+            Assert.AreEqual(2, updated);
+            Assert.AreEqual(2, personQueryable.Count(p => p.Value.Name == name));
+        }
+
+        /// <summary>
+        /// Tests the UpdateAll extension with condition.
+        /// </summary>
+        [Test]
+        public void TestUpdateAllWithCondition()
+        {
+            // ReSharper disable AccessToModifiedClosure
+            // Use new cache to avoid touching static data.
+            var personQueryable = GetPersonCacheQueryable("updateAllTest_WithCondition_Persons", 10);
+
+            // Simple conditional
+            var aliasValue = 777;
+            var updated = personQueryable
+                .Where(p => p.Key > 8)
+                .UpdateAll(d => d.Set(p => p.AliasTest, aliasValue));
+            Assert.AreEqual(2, updated);
+            AssertAll(personQueryable,
+                p => p.Key <= 8 && p.Value.AliasTest != aliasValue || p.Value.AliasTest == aliasValue);
+
+            // Conditional with limit
+            aliasValue = 8888;
+            updated = personQueryable
+                .Where(p => p.Key > 1)
+                .Take(3)
+                .UpdateAll(d => d.Set(p => p.AliasTest, aliasValue));
+            Assert.AreEqual(3, updated);
+            Assert.AreEqual(3, personQueryable.ToArray().Count(p => p.Value.AliasTest == aliasValue));
+            // ReSharper restore AccessToModifiedClosure
+        }
+
+        /// <summary>
+        /// Tests not supported queries for the UpdateAll extension .
+        /// </summary>
+        [Test]
+        public void TestUpdateAllUnsupported()
+        {
+            // Use new cache to avoid touching static data.
+            var personQueryable = GetPersonCacheQueryable("updateAllTest_Unsupported_Persons", 10);
+            
+            // Skip is not supported with DELETE.
+            var nex = Assert.Throws<NotSupportedException>(
+                () => personQueryable.Skip(1).UpdateAll(d => d.Set(p => p.Age, 15)));
+            Assert.AreEqual("UpdateAll can not be combined with result operators (other than Take): SkipResultOperator",
+                nex.Message);
+
+            // Multiple result operators are not supported with DELETE.
+            nex = Assert.Throws<NotSupportedException>(() =>
+                personQueryable.Skip(1).Take(1).UpdateAll(d => d.Set(p => p.Age, 15)));
+            Assert.AreEqual(
+                "UpdateAll can not be combined with result operators (other than Take): SkipResultOperator, " +
+                "TakeResultOperator, UpdateAllResultOperator", nex.Message);
+
+            // Joins are not supported in H2.
+            var qry = personQueryable
+                .Where(x => x.Key == 7)
+                .Join(GetPersonCache().AsCacheQueryable(), p => p.Key, p => p.Key, (p1, p2) => p1);
+
+            var ex = Assert.Throws<IgniteException>(() => qry.UpdateAll(d => d.Set(p => p.Age, 15)));
+            Assert.AreEqual("Failed to parse query", ex.Message.Substring(0, 21));
+        }
+
+        /// <summary>
+        /// Gets filled persons cache  queryable
+        /// </summary>
+        private IQueryable<ICacheEntry<int, Person>> GetPersonCacheQueryable(string cacheName, int personCount,
+            int? orgCount = null)
+        {
+            var cache = GetCache<Person>(cacheName);
+
+            cache.PutAll(Enumerable.Range(1, personCount).ToDictionary(x => x,
+                x => new Person(x, x.ToString())
+                {
+                    Birthday = DateTime.UtcNow.AddDays(personCount - x),
+                    OrganizationId = x % orgCount ?? 0
+                }));
+
+            return cache.AsCacheQueryable();
+        }
+
+        /// <summary>
+        /// Asserts that all values in cache correspond to predicate
+        /// </summary>
+        private static void AssertAll(
+            // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Local
+            IQueryable<ICacheEntry<int, Person>> personQueryable, 
+            // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Local
+            Func<ICacheEntry<int, Person>, bool> predicate)
+        {
+            Assert.IsTrue(personQueryable.ToArray().All(predicate));
+        }
+
+        /// <summary>
+        /// Gets filled organization cache queryable
+        /// </summary>
+        private IQueryable<ICacheEntry<int, Organization>> GetOrgCacheQueryable(string cacheName, int orgCount)
+        {
+            var cache = GetCache<Organization>(cacheName);
+            var allOrg = Enumerable.Range(1, orgCount)
+                .Select(x => new Organization
+                {
+                    Id = x,
+                    Name = x.ToString()
+                })
+                .ToList();
+
+            allOrg.ForEach(x => cache.Put(x.Id, x));
+
+            return cache.AsCacheQueryable();
+        }
+
+        /// <summary>
+        /// Gets cache of <see cref="T"/>
+        /// </summary>
+        private ICache<int, T> GetCache<T>(string cacheName)
+        {
+            return Ignition.GetIgnite().GetOrCreateCache<int, T>(new CacheConfiguration(cacheName,
+                new QueryEntity(typeof(int), typeof(T)))
+            {
+                SqlEscapeAll = GetSqlEscapeAll(),
+                CacheMode = CacheMode.Replicated
+            });
+        }
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.csproj
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.csproj b/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.csproj
index fc13914..d5a2d83 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.csproj
@@ -64,7 +64,10 @@
     <Compile Include="Impl\CacheQueryExpressionVisitor.cs" />
     <Compile Include="Impl\CacheQueryModelVisitor.cs" />
     <Compile Include="Impl\CacheQueryParser.cs" />
+    <Compile Include="Impl\Dml\MemberUpdateContainer.cs" />
+    <Compile Include="Impl\Dml\UpdateAllExpressionNode.cs" />
     <Compile Include="Impl\Dml\RemoveAllExpressionNode.cs" />
+    <Compile Include="Impl\Dml\UpdateAllResultOperator.cs" />
     <Compile Include="Impl\Dml\RemoveAllResultOperator.cs" />
     <Compile Include="Impl\EnumerableHelper.cs" />
     <Compile Include="Impl\ICacheQueryableInternal.cs" />
@@ -73,6 +76,7 @@
     <Compile Include="Impl\SqlTypes.cs" />
     <Compile Include="Impl\ExpressionWalker.cs" />
     <Compile Include="Impl\JoinInnerSequenceParameterNotNullExpressionVisitor.cs" />
+    <Compile Include="IUpdateDescriptor.cs" />
     <Compile Include="Package-Info.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="QueryOptions.cs" />

http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs
index 935666f..2b65d85 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs
@@ -193,5 +193,33 @@ namespace Apache.Ignite.Linq
             return query.Provider.Execute<int>(Expression.Call(null, method, query.Expression,
                 Expression.Quote(predicate)));
         }
+
+        /// <summary>
+        /// Updates all rows that are matched by the specified query.
+        /// <para />
+        /// This method results in "UPDATE" distributed SQL query, performing bulk update 
+        /// (as opposed to fetching all rows locally).
+        /// </summary>
+        /// <typeparam name="TKey">Key type.</typeparam>
+        /// <typeparam name="TValue">Value type.</typeparam>
+        /// <param name="query">The query.</param>
+        /// <param name="updateDescription">The update description.</param>
+        /// <returns>Affected row count.</returns>
+        public static int UpdateAll<TKey, TValue>(this IQueryable<ICacheEntry<TKey, TValue>> query,
+            Expression<Func<IUpdateDescriptor<TKey,TValue>, IUpdateDescriptor<TKey,TValue>>> updateDescription)
+        {
+            IgniteArgumentCheck.NotNull(query, "query");
+            IgniteArgumentCheck.NotNull(updateDescription, "updateDescription");
+
+            var method = UpdateAllExpressionNode.UpdateAllMethodInfo
+                .MakeGenericMethod(typeof(TKey), typeof(TValue)); // TODO: cache?
+
+            return query.Provider.Execute<int>(Expression.Call(null, method, new[]
+            {
+                query.Expression,
+                Expression.Quote(updateDescription)
+            }));
+        }
+
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/IUpdateDescriptor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/IUpdateDescriptor.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/IUpdateDescriptor.cs
new file mode 100644
index 0000000..2327b99
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/IUpdateDescriptor.cs
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Linq
+{
+    using System;
+    using Apache.Ignite.Core.Cache;
+
+    /// <summary>
+    /// Dummy interface to provide update description for
+    /// <see
+    ///     cref="CacheLinqExtensions.UpdateAll{TKey,TValue}" />
+    /// </summary>
+    /// <typeparam name="TKey">Key type.</typeparam>
+    /// <typeparam name="TValue">Value type.</typeparam>
+    public interface IUpdateDescriptor<out TKey, out TValue>
+    {
+        /// <summary>
+        /// Specifies member update with constant
+        /// </summary>
+        /// <typeparam name="TProp">Member type</typeparam>
+        /// <param name="selector">Member selector</param>
+        /// <param name="value">New value</param>
+        /// <returns></returns>
+        IUpdateDescriptor<TKey, TValue> Set<TProp>(Func<TValue, TProp> selector, TProp value);
+
+        /// <summary>
+        /// Specifies member update with expression
+        /// </summary>
+        /// <typeparam name="TProp">Member type</typeparam>
+        /// <param name="selector">Member selector</param>
+        /// <param name="valueBuilder">New value generator</param>
+        /// <returns></returns>
+        IUpdateDescriptor<TKey, TValue> Set<TProp>(Func<TValue, TProp> selector,
+            Func<ICacheEntry<TKey, TValue>, TProp> valueBuilder);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/AliasDictionary.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/AliasDictionary.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/AliasDictionary.cs
index 8902e7c..15710db 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/AliasDictionary.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/AliasDictionary.cs
@@ -20,6 +20,7 @@ namespace Apache.Ignite.Linq.Impl
     using System;
     using System.Collections.Generic;
     using System.Diagnostics;
+    using System.Linq;
     using System.Linq.Expressions;
     using System.Text;
     using Remotion.Linq.Clauses;
@@ -49,11 +50,14 @@ namespace Apache.Ignite.Linq.Impl
         /// <summary>
         /// Pushes current aliases to stack.
         /// </summary>
-        public void Push()
+        /// <param name="copyAliases">Flag indicating that current aliases should be copied</param>
+        public void Push(bool copyAliases)
         {
             _stack.Push(_tableAliases);
 
-            _tableAliases = new Dictionary<IQuerySource, string>();
+            _tableAliases = copyAliases
+                ? _tableAliases.ToDictionary(p => p.Key, p => p.Value)
+                : new Dictionary<IQuerySource, string>();
         }
 
         /// <summary>

http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs
index 4caefe1..c5c887f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs
@@ -54,21 +54,34 @@ namespace Apache.Ignite.Linq.Impl
         /** */
         private readonly bool _includeAllFields;
 
+        /** */
+        private readonly bool _visitEntireSubQueryModel;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="CacheQueryExpressionVisitor" /> class.
         /// </summary>
         /// <param name="modelVisitor">The _model visitor.</param>
-        /// <param name="useStar">Flag indicating that star '*' qualifier should be used
-        /// for the whole-table select instead of _key, _val.</param>
-        /// <param name="includeAllFields">Flag indicating that star '*' qualifier should be used
-        /// for the whole-table select as well as _key, _val.</param>
-        public CacheQueryExpressionVisitor(CacheQueryModelVisitor modelVisitor, bool useStar, bool includeAllFields)
+        /// <param name="useStar">
+        /// Flag indicating that star '*' qualifier should be used
+        /// for the whole-table select instead of _key, _val.
+        /// </param>
+        /// <param name="includeAllFields">
+        /// Flag indicating that star '*' qualifier should be used
+        /// for the whole-table select as well as _key, _val.
+        /// </param>
+        /// <param name="visitEntireSubQueryModel">
+        /// Flag indicating that subquery 
+        /// should be visited as full query
+        /// </param>
+        public CacheQueryExpressionVisitor(CacheQueryModelVisitor modelVisitor, bool useStar, bool includeAllFields,
+            bool visitEntireSubQueryModel)
         {
             Debug.Assert(modelVisitor != null);
 
             _modelVisitor = modelVisitor;
             _useStar = useStar;
             _includeAllFields = includeAllFields;
+            _visitEntireSubQueryModel = visitEntireSubQueryModel;
         }
 
         /// <summary>
@@ -548,12 +561,18 @@ namespace Apache.Ignite.Linq.Impl
             var subQueryModel = expression.QueryModel;
 
             var contains = subQueryModel.ResultOperators.FirstOrDefault() as ContainsResultOperator;
-            
+
             // Check if IEnumerable.Contains is used.
-            if (subQueryModel.ResultOperators.Count == 1 && contains != null)
+            if (subQueryModel.ResultOperators.Count == 1 && contains != null) 
             {
                 VisitContains(subQueryModel, contains);
             }
+            else if (_visitEntireSubQueryModel)
+            {
+                ResultBuilder.Append("(");
+                _modelVisitor.VisitQueryModel(subQueryModel, false, true);
+                ResultBuilder.Append(")");
+            }
             else
             {
                 // This happens when New expression uses a subquery, in a GroupBy.
@@ -579,7 +598,15 @@ namespace Apache.Ignite.Linq.Impl
                 Visit(contains.Item);
 
                 ResultBuilder.Append(" IN (");
-                _modelVisitor.VisitQueryModel(subQueryModel);
+                if (_visitEntireSubQueryModel)
+                {
+                    _modelVisitor.VisitQueryModel(subQueryModel, false, true);
+                }
+                else
+                {
+                    _modelVisitor.VisitQueryModel(subQueryModel);
+                }
+                
                 ResultBuilder.Append(")");
             }
             else

http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
index 223eea6..da250e9 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
@@ -103,26 +103,30 @@ namespace Apache.Ignite.Linq.Impl
         /// <summary>
         /// Visits the query model.
         /// </summary>
-        private void VisitQueryModel(QueryModel queryModel, bool includeAllFields)
+        internal void VisitQueryModel(QueryModel queryModel, bool includeAllFields, bool copyAliases = false)
         {
-            _aliases.Push();
+            _aliases.Push(copyAliases);
 
-            var hasDelete = VisitRemoveOperator(queryModel);
-
-            if (!hasDelete)
+            var lastResultOp = queryModel.ResultOperators.LastOrDefault();
+            if (lastResultOp is RemoveAllResultOperator)
+            {
+                VisitRemoveOperator(queryModel);
+            }
+            else if(lastResultOp is UpdateAllResultOperator)
+            {
+                VisitUpdateAllOperator(queryModel);
+            }
+            else
             {
                 // SELECT
                 _builder.Append("select ");
 
                 // TOP 1 FLD1, FLD2
                 VisitSelectors(queryModel, includeAllFields);
-            }
 
-            // FROM ... WHERE ... JOIN ...
-            base.VisitQueryModel(queryModel);
+                // FROM ... WHERE ... JOIN ...
+                base.VisitQueryModel(queryModel);
 
-            if (!hasDelete)
-            {
                 // UNION ...
                 ProcessResultOperatorsEnd(queryModel);
             }
@@ -131,42 +135,68 @@ namespace Apache.Ignite.Linq.Impl
         }
 
         /// <summary>
-        /// Visits the remove operator. Returns true if it is present.
+        /// Visits the remove operator.
         /// </summary>
-        private bool VisitRemoveOperator(QueryModel queryModel)
+        private void VisitRemoveOperator(QueryModel queryModel)
         {
             var resultOps = queryModel.ResultOperators;
 
-            if (resultOps.LastOrDefault() is RemoveAllResultOperator)
+            _builder.Append("delete ");
+
+            if (resultOps.Count == 2)
             {
-                _builder.Append("delete ");
+                var resOp = resultOps[0] as TakeResultOperator;
 
-                if (resultOps.Count == 2)
-                {
-                    var resOp = resultOps[0] as TakeResultOperator;
+                if (resOp == null)
+                    throw new NotSupportedException(
+                        "RemoveAll can not be combined with result operators (other than Take): " +
+                        resultOps[0].GetType().Name);
 
-                    if (resOp == null)
-                    {
-                        throw new NotSupportedException(
-                            "RemoveAll can not be combined with result operators (other than Take): " +
-                            resultOps[0].GetType().Name);
-                    }
+                _builder.Append("top ");
+                BuildSqlExpression(resOp.Count);
+                _builder.Append(" ");
+            }
+            else if (resultOps.Count > 2)
+            {
+                throw new NotSupportedException(
+                    "RemoveAll can not be combined with result operators (other than Take): " +
+                    string.Join(", ", resultOps.Select(x => x.GetType().Name)));
+            }
 
-                    _builder.Append("top ");
-                    BuildSqlExpression(resOp.Count);
-                    _builder.Append(" ");
-                }
-                else if (resultOps.Count > 2)
-                {
+            // FROM ... WHERE ... JOIN ...
+            base.VisitQueryModel(queryModel);
+        }
+
+        /// <summary>
+        /// Visits the UpdateAll operator.
+        /// </summary>
+        private void VisitUpdateAllOperator(QueryModel queryModel)
+        {
+            var resultOps = queryModel.ResultOperators;
+
+            _builder.Append("update ");
+
+            // FROM ... WHERE ...
+            base.VisitQueryModel(queryModel);
+
+            if (resultOps.Count == 2)
+            {
+                var resOp = resultOps[0] as TakeResultOperator;
+
+                if (resOp == null)
                     throw new NotSupportedException(
-                        "RemoveAll can not be combined with result operators (other than Take): " +
-                        string.Join(", ", resultOps.Select(x => x.GetType().Name)));
-                }
+                        "UpdateAll can not be combined with result operators (other than Take): " +
+                        resultOps[0].GetType().Name);
 
-                return true;
+                _builder.Append("limit ");
+                BuildSqlExpression(resOp.Count);
+            }
+            else if (resultOps.Count > 2)
+            {
+                throw new NotSupportedException(
+                    "UpdateAll can not be combined with result operators (other than Take): " +
+                    string.Join(", ", resultOps.Select(x => x.GetType().Name)));
             }
-                
-            return false;
         }
 
         /// <summary>
@@ -394,18 +424,28 @@ namespace Apache.Ignite.Linq.Impl
         {
             base.VisitMainFromClause(fromClause, queryModel);
 
-            _builder.AppendFormat("from ");
+            var isUpdateQuery = queryModel.ResultOperators.LastOrDefault() is UpdateAllResultOperator;
+            if (!isUpdateQuery)
+            {
+                _builder.Append("from ");
+            }
+
             ValidateFromClause(fromClause);
             _aliases.AppendAsClause(_builder, fromClause).Append(" ");
 
             var i = 0;
             foreach (var additionalFrom in queryModel.BodyClauses.OfType<AdditionalFromClause>())
             {
-                _builder.AppendFormat(", ");
+                _builder.Append(", ");
                 ValidateFromClause(additionalFrom);
 
                 VisitAdditionalFromClause(additionalFrom, queryModel, i++);
             }
+
+            if (isUpdateQuery)
+            {
+                BuildSetClauseForUpdateAll(queryModel);
+            }
         }
 
         /// <summary>
@@ -535,7 +575,6 @@ namespace Apache.Ignite.Linq.Impl
             }
         }
 
-
         /// <summary>
         /// Visists Join clause in case of join with local collection
         /// </summary>
@@ -651,9 +690,33 @@ namespace Apache.Ignite.Linq.Impl
         /// <summary>
         /// Builds the SQL expression.
         /// </summary>
-        private void BuildSqlExpression(Expression expression, bool useStar = false, bool includeAllFields = false)
+        private void BuildSqlExpression(Expression expression, bool useStar = false, bool includeAllFields = false,
+            bool visitSubqueryModel = false)
+        {
+            new CacheQueryExpressionVisitor(this, useStar, includeAllFields, visitSubqueryModel).Visit(expression);
+        }
+
+        /// <summary>
+        /// Builds SET clause of UPDATE statement
+        /// </summary>
+        private void BuildSetClauseForUpdateAll(QueryModel queryModel)
         {
-            new CacheQueryExpressionVisitor(this, useStar, includeAllFields).Visit(expression);
+            var updateAllResultOperator = queryModel.ResultOperators.LastOrDefault() as UpdateAllResultOperator;
+            if (updateAllResultOperator != null)
+            {
+                _builder.Append("set ");
+                var first = true;
+                foreach (var update in updateAllResultOperator.Updates)
+                {
+                    if (!first) _builder.Append(", ");
+                    first = false;
+                    BuildSqlExpression(update.Selector);
+                    _builder.Append(" = ");
+                    BuildSqlExpression(update.Value, visitSubqueryModel: true);
+                }
+
+                _builder.Append(" ");
+            }
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryParser.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryParser.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryParser.cs
index 17ec0a3..3f69cce 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryParser.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryParser.cs
@@ -66,6 +66,9 @@ namespace Apache.Ignite.Linq.Impl
             methodInfoRegistry.Register(RemoveAllExpressionNode.GetSupportedMethods(), 
                 typeof(RemoveAllExpressionNode));
 
+            methodInfoRegistry.Register(UpdateAllExpressionNode.GetSupportedMethods(),
+                typeof(UpdateAllExpressionNode));
+
             return new CompoundNodeTypeProvider(new INodeTypeProvider[]
             {
                 methodInfoRegistry,

http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/MemberUpdateContainer.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/MemberUpdateContainer.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/MemberUpdateContainer.cs
new file mode 100644
index 0000000..bf8a5fd
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/MemberUpdateContainer.cs
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+namespace Apache.Ignite.Linq.Impl.Dml
+{
+    using System.Linq.Expressions;
+
+    /// <summary>
+    /// Contains information about member update
+    /// </summary>
+    internal struct MemberUpdateContainer
+    {
+        /// <summary>
+        /// Gets or sets member selector
+        /// </summary>
+        public Expression Selector { get; set; }
+
+        /// <summary>
+        /// Gets or sets member new value
+        /// </summary>
+        public Expression Value { get; set; }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllExpressionNode.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllExpressionNode.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllExpressionNode.cs
new file mode 100644
index 0000000..8cfb552
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllExpressionNode.cs
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Linq.Impl.Dml
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Diagnostics.CodeAnalysis;
+    using System.Linq;
+    using System.Linq.Expressions;
+    using System.Reflection;
+    using Remotion.Linq.Clauses;
+    using Remotion.Linq.Clauses.Expressions;
+    using Remotion.Linq.Parsing.ExpressionVisitors;
+    using Remotion.Linq.Parsing.Structure.IntermediateModel;
+
+    /// <summary>
+    /// Represents a <see cref="MethodCallExpression" /> for
+    /// <see cref="CacheLinqExtensions.UpdateAll{TKey,TValue}" />.
+    /// When user calls UpdateAll, this node is generated.
+    /// </summary>
+    internal sealed class UpdateAllExpressionNode : ResultOperatorExpressionNodeBase
+    {
+        /// <summary>
+        /// The UpdateAll method.
+        /// </summary>
+        public static readonly MethodInfo UpdateAllMethodInfo = typeof(CacheLinqExtensions)
+            .GetMethods()
+            .Single(x => x.Name == "UpdateAll");
+
+        //** */
+        private static readonly MethodInfo[] SupportedMethods = {UpdateAllMethodInfo};
+
+        //** */
+        private readonly LambdaExpression _updateDescription;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="UpdateAllExpressionNode" /> class.
+        /// </summary>
+        /// <param name="parseInfo">The parse information.</param>
+        /// <param name="updateDescription">Expression with update description info</param>
+        public UpdateAllExpressionNode(MethodCallExpressionParseInfo parseInfo,
+            LambdaExpression updateDescription)
+            : base(parseInfo, null, null)
+        {
+            _updateDescription = updateDescription;
+        }
+
+        /** <inheritdoc /> */
+        [ExcludeFromCodeCoverage]
+        public override Expression Resolve(ParameterExpression inputParameter, Expression expressionToBeResolved,
+            ClauseGenerationContext clauseGenerationContext)
+        {
+            throw CreateResolveNotSupportedException();
+        }
+
+        /** <inheritdoc /> */
+        protected override ResultOperatorBase CreateResultOperator(ClauseGenerationContext clauseGenerationContext)
+        {
+            if (_updateDescription.Parameters.Count != 1)
+                throw new NotSupportedException("Expression is not supported for UpdateAll: " +
+                                                _updateDescription);
+
+            var querySourceRefExpression = (QuerySourceReferenceExpression) Source.Resolve(
+                _updateDescription.Parameters[0],
+                _updateDescription.Parameters[0],
+                clauseGenerationContext);
+
+            var cacheEntryType = querySourceRefExpression.Type;
+            var querySourceAccessValue =
+                Expression.MakeMemberAccess(querySourceRefExpression, cacheEntryType.GetMember("Value").First());
+
+            if (!(_updateDescription.Body is MethodCallExpression))
+                throw new NotSupportedException("Expression is not supported for UpdateAll: " +
+                                                _updateDescription.Body);
+
+            var updates = new List<MemberUpdateContainer>();
+
+            var methodCall = (MethodCallExpression) _updateDescription.Body;
+            while (methodCall != null)
+            {
+                if (methodCall.Arguments.Count != 2)
+                    throw new NotSupportedException("Method is not supported for UpdateAll: " + methodCall);
+
+                var selectorLambda = (LambdaExpression) methodCall.Arguments[0];
+                var selector = ReplacingExpressionVisitor.Replace(selectorLambda.Parameters[0], querySourceAccessValue,
+                    selectorLambda.Body);
+
+                var newValue = methodCall.Arguments[1];
+                switch (newValue.NodeType)
+                {
+                    case ExpressionType.Constant:
+                        break;
+                    case ExpressionType.Lambda:
+                        var newValueLambda = (LambdaExpression) newValue;
+                        newValue = ReplacingExpressionVisitor.Replace(newValueLambda.Parameters[0],
+                            querySourceRefExpression, newValueLambda.Body);
+                        break;
+                    default:
+                        throw new NotSupportedException("Value expression is not supported for UpdateAll: "
+                                                        + newValue);
+                }
+
+                updates.Add(new MemberUpdateContainer
+                {
+                    Selector = selector,
+                    Value = newValue
+                });
+
+                methodCall = methodCall.Object as MethodCallExpression;
+            }
+
+            return new UpdateAllResultOperator(updates);
+        }
+
+        /// <summary>
+        /// Gets the supported methods.
+        /// </summary>
+        public static IEnumerable<MethodInfo> GetSupportedMethods()
+        {
+            return SupportedMethods;
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllResultOperator.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllResultOperator.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllResultOperator.cs
new file mode 100644
index 0000000..be682b0
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllResultOperator.cs
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Linq.Impl.Dml
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Diagnostics.CodeAnalysis;
+    using System.Linq;
+    using System.Linq.Expressions;
+    using Remotion.Linq.Clauses;
+    using Remotion.Linq.Clauses.ResultOperators;
+    using Remotion.Linq.Clauses.StreamedData;
+
+    /// <summary>
+    /// Represents an operator for <see cref="CacheLinqExtensions.UpdateAll{TKey,TValue}"/>.
+    /// </summary>
+    internal sealed class UpdateAllResultOperator : ValueFromSequenceResultOperatorBase
+    {
+        /// <summary>
+        /// Gets the members updates
+        /// </summary>
+        public MemberUpdateContainer[] Updates { get; private set; }
+
+        /// <summary>
+        /// Initializes a new instance of <see cref="UpdateAllResultOperator" />
+        /// </summary>
+        /// <param name="updates">members updates</param>
+        public UpdateAllResultOperator(IEnumerable<MemberUpdateContainer> updates)
+        {
+            Updates = updates.ToArray();
+        }
+
+        /** <inheritdoc /> */
+        public override IStreamedDataInfo GetOutputDataInfo(IStreamedDataInfo inputInfo)
+        {
+            return new StreamedScalarValueInfo(typeof(int));
+        }
+
+        /** <inheritdoc /> */
+        [ExcludeFromCodeCoverage]
+        public override ResultOperatorBase Clone(CloneContext cloneContext)
+        {
+            return new UpdateAllResultOperator(Updates);
+        }
+
+        /** <inheritdoc /> */
+        [ExcludeFromCodeCoverage]
+        public override void TransformExpressions(Func<Expression, Expression> transformation)
+        {
+            // No-op.
+        }
+
+        /** <inheritdoc /> */
+        [ExcludeFromCodeCoverage]
+        public override StreamedValue ExecuteInMemory<T>(StreamedSequence sequence)
+        {
+            throw new NotSupportedException("UpdateAll is not supported for in-memory sequences.");
+        }
+    }
+}
\ No newline at end of file


[21/50] [abbrv] ignite git commit: IGNITE-7765 Ensure that walSegmentSize doesn't overflow.

Posted by sb...@apache.org.
IGNITE-7765 Ensure that walSegmentSize doesn't overflow.

Signed-off-by: Andrey Gura <ag...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8da7c9e1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8da7c9e1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8da7c9e1

Branch: refs/heads/ignite-zk
Commit: 8da7c9e1b90d14fbce971a6566ce369b795ed30b
Parents: 1f8ec79
Author: Ivan Daschinskiy <iv...@gmail.com>
Authored: Tue Mar 13 17:32:02 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Mar 13 17:32:30 2018 +0300

----------------------------------------------------------------------
 .../configuration/DataStorageConfiguration.java |  8 ++--
 .../DataStorageConfigurationValidationTest.java | 46 ++++++++++++++++++++
 .../ignite/testsuites/IgniteCacheTestSuite.java |  2 +
 3 files changed, 53 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/8da7c9e1/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
index 4a3dbbf..a433760 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
@@ -528,16 +528,18 @@ public class DataStorageConfiguration implements Serializable {
      * @return WAL segment size.
      */
     public int getWalSegmentSize() {
-        return walSegmentSize <= 0 ? DFLT_WAL_SEGMENT_SIZE : walSegmentSize;
+        return walSegmentSize == 0 ? DFLT_WAL_SEGMENT_SIZE : walSegmentSize;
     }
 
     /**
      * Sets size of a WAL segment.
      *
-     * @param walSegmentSize WAL segment size. 64 MB is used by default.  Maximum value is 2Gb
-     * @return {@code this} for chaining.
+     * @param walSegmentSize WAL segment size. 64 MB is used by default.  Maximum value is 2Gb.
+     * @return {@code This} for chaining.
      */
     public DataStorageConfiguration setWalSegmentSize(int walSegmentSize) {
+        A.ensure(walSegmentSize >= 0, "WAL segment size must be non-negative and less than 2 Gb.");
+
         this.walSegmentSize = walSegmentSize;
 
         return this;

http://git-wip-us.apache.org/repos/asf/ignite/blob/8da7c9e1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java
new file mode 100644
index 0000000..7f667ee
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.concurrent.Callable;
+import junit.framework.TestCase;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Tests assertions in DataStorageConfiguration.
+ */
+public class DataStorageConfigurationValidationTest extends TestCase {
+    /**
+     * Tests {@link DataStorageConfiguration#walSegmentSize} property assertion.
+     *
+     * @throws Exception If failed.
+     */
+    public void testWalSegmentSizeOveflow() throws Exception {
+        final DataStorageConfiguration cfg = new DataStorageConfiguration();
+
+        GridTestUtils.assertThrows(null, new Callable<Void>() {
+            /** {@inheritDoc} */
+            @Override public Void call() {
+                cfg.setWalSegmentSize(1 << 31);
+
+                return null;
+            }
+        }, IllegalArgumentException.class, null);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/8da7c9e1/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index b309b0a..bb397f7 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -82,6 +82,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheSwapPreloadSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheTtlManagerSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheTxPartitionedLocalStoreSelfTest;
 import org.apache.ignite.internal.processors.cache.GridDataStorageConfigurationConsistencySelfTest;
+import org.apache.ignite.internal.processors.cache.DataStorageConfigurationValidationTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicInvokeTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalInvokeTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalWithStoreInvokeTest;
@@ -229,6 +230,7 @@ public class IgniteCacheTestSuite extends TestSuite {
         suite.addTestSuite(GridCacheConfigurationValidationSelfTest.class);
         suite.addTestSuite(GridCacheConfigurationConsistencySelfTest.class);
         suite.addTestSuite(GridDataStorageConfigurationConsistencySelfTest.class);
+        suite.addTestSuite(DataStorageConfigurationValidationTest.class);
         suite.addTestSuite(GridCacheJdbcBlobStoreSelfTest.class);
         suite.addTestSuite(GridCacheJdbcBlobStoreMultithreadedSelfTest.class);
         suite.addTestSuite(JdbcTypesDefaultTransformerTest.class);


[14/50] [abbrv] ignite git commit: IGNITE-7878 .NET: Un-Ignore TestStartDefault

Posted by sb...@apache.org.
IGNITE-7878 .NET: Un-Ignore TestStartDefault

This reverts commit 18c6b4a5d079874e9c156fa39564970ad3c31914.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ed2435c8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ed2435c8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ed2435c8

Branch: refs/heads/ignite-zk
Commit: ed2435c81e435750fb2640aac1ea2a9ae40bce2d
Parents: 318ffe5
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Mon Mar 12 16:21:41 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Mon Mar 12 16:21:41 2018 +0300

----------------------------------------------------------------------
 .../dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs          | 1 -
 1 file changed, 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/ed2435c8/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
index 5ef095c..792b33d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
@@ -47,7 +47,6 @@ namespace Apache.Ignite.Core.Tests
         /// 
         /// </summary>
         [Test]
-        [Ignore("IGNITE-7878")]
         public void TestStartDefault()
         {
             var cfg = new IgniteConfiguration {JvmClasspath = TestUtils.CreateTestClasspath()};


[29/50] [abbrv] ignite git commit: IGNITE-7947 Not all OWNING partitions saved in PartitionAllocationMap during checkpoint - Fixes #3632.

Posted by sb...@apache.org.
IGNITE-7947 Not all OWNING partitions saved in PartitionAllocationMap during checkpoint - Fixes #3632.

Signed-off-by: Alexey Goncharuk <al...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/798c01bd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/798c01bd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/798c01bd

Branch: refs/heads/ignite-zk
Commit: 798c01bd6d8427ec85ebbef4c0bec4fdcacfee29
Parents: 48a6623
Author: Eduard Shangareev <es...@gridgain.com>
Authored: Wed Mar 14 16:48:23 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Mar 14 16:48:23 2018 +0300

----------------------------------------------------------------------
 .../preloader/GridDhtPartitionExchangeId.java   |  3 ++
 .../persistence/GridCacheOffheapManager.java    | 41 ++++++++++++++------
 2 files changed, 33 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/798c01bd/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java
index 33728d3..741386b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java
@@ -212,6 +212,9 @@ public class GridDhtPartitionExchangeId implements Message, Comparable<GridDhtPa
 
     /** {@inheritDoc} */
     @Override public boolean equals(Object o) {
+        if (o == null)
+            return false;
+
         if (o == this)
             return true;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/798c01bd/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index eb0c30f..d3b73bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -51,7 +51,6 @@ import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteHistoricalIterator;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
@@ -73,7 +72,6 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.query.GridQueryRowCacheCleaner;
 import org.apache.ignite.internal.util.lang.GridCursor;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -162,14 +160,8 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
 
         boolean metaWasUpdated = false;
 
-        for (CacheDataStore store : partDataStores.values()) {
-            RowStore rowStore = store.rowStore();
-
-            if (rowStore == null)
-                continue;
-
+        for (CacheDataStore store : partDataStores.values())
             metaWasUpdated |= saveStoreMetadata(store, ctx, !metaWasUpdated, false);
-        }
     }
 
     /**
@@ -210,8 +202,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
                     if (beforeDestroy)
                         state = GridDhtPartitionState.EVICTED;
                     else {
-                        part = grp.topology().localPartition(store.partId(),
-                                AffinityTopologyVersion.NONE, false, true);
+                        part = getPartition(store);
 
                         if (part != null && part.state() != GridDhtPartitionState.EVICTED)
                             state = part.state();
@@ -337,12 +328,40 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
                     pageMem.releasePage(grpId, partMetaId, partMetaPage);
                 }
             }
+            else if (needSnapshot)
+                tryAddEmptyPartitionToSnapshot(store, ctx);;
         }
+        else if (needSnapshot)
+            tryAddEmptyPartitionToSnapshot(store, ctx);
 
         return wasSaveToMeta;
     }
 
     /**
+     * Check that we need to snapshot this partition and add it to map.
+     *
+     * @param store Store.
+     * @param ctx Snapshot context.
+     */
+    private void tryAddEmptyPartitionToSnapshot(CacheDataStore store, Context ctx) {
+        if (getPartition(store).state() == OWNING) {
+            ctx.partitionStatMap().put(
+                    new GroupPartitionId(grp.groupId(), store.partId()),
+                    new PagesAllocationRange(0, 0));
+        }
+    }
+
+    /**
+     * @param store Store.
+     *
+     * @return corresponding to store local partition
+     */
+    private GridDhtLocalPartition getPartition(CacheDataStore store) {
+        return grp.topology().localPartition(store.partId(),
+                AffinityTopologyVersion.NONE, false, true);
+    }
+
+    /**
      * Loads cache sizes for all caches in shared group.
      *
      * @param pageMem page memory to perform operations on pages.


[16/50] [abbrv] ignite git commit: .NET: Update DEVNOTES - better .NET Core test commands

Posted by sb...@apache.org.
.NET: Update DEVNOTES - better .NET Core test commands


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3cb9e258
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3cb9e258
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3cb9e258

Branch: refs/heads/ignite-zk
Commit: 3cb9e2582470dbb4ace357e45d48158a3ccba642
Parents: 447a8a9
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Mon Mar 12 20:12:13 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Mon Mar 12 20:12:13 2018 +0300

----------------------------------------------------------------------
 modules/platforms/dotnet/DEVNOTES.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/3cb9e258/modules/platforms/dotnet/DEVNOTES.txt
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/DEVNOTES.txt b/modules/platforms/dotnet/DEVNOTES.txt
index 0726c46..199fe90 100644
--- a/modules/platforms/dotnet/DEVNOTES.txt
+++ b/modules/platforms/dotnet/DEVNOTES.txt
@@ -49,9 +49,9 @@ Getting started:
   ./build.sh
 * Run tests:
   cd Apache.Ignite.Core.Tests.DotNetCore
-  dotnet test
+  dotnet test --logger "console;verbosity=normal"
 * Run specific test:
-  dotnet test --filter CacheTest
+  dotnet test --filter CacheTest --logger "console;verbosity=normal"
 * IDE: Open Apache.Ignite.DotNetCore.sln
 
 


[19/50] [abbrv] ignite git commit: IGNITE-7638 Implemented robin-hood hashing for FullPageIdTable - Fixes #3481.

Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
new file mode 100644
index 0000000..fd4e0e6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
@@ -0,0 +1,654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.util.function.BiConsumer;
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.util.GridLongList;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.NotNull;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR;
+import static org.apache.ignite.IgniteSystemProperties.getFloat;
+import static org.apache.ignite.internal.util.GridUnsafe.getInt;
+import static org.apache.ignite.internal.util.GridUnsafe.getLong;
+import static org.apache.ignite.internal.util.GridUnsafe.putInt;
+import static org.apache.ignite.internal.util.GridUnsafe.putLong;
+
+/**
+ * Loaded pages mapping to relative pointer based on Robin Hood hashing: backward shift deletion algorithm. <br>
+ * Performance of initial Robin Hood hashing could be greatly improved with only a little change to the removal
+ * method.<br> Instead of replacing entries with 'Removed' fake entries on deletion, backward shift deletion variant
+ * for the Robin Hood hashing algorithm does shift backward all the entries following the entry to delete until
+ * either an empty bucket, or a bucket with a DIB of 0 (distance to initial bucket).<br>
+ *
+ * Every deletion will shift backwards entries and therefore decrease their respective DIBs by 1
+ * (all their initial DIB values would be >= 1)<br>
+ *
+ * This implementation stores ideal bucket with entry value itself.<br>
+ *
+ */
+public class RobinHoodBackwardShiftHashMap implements LoadedPagesMap {
+    /** Load factor. */
+    private static final float LOAD_FACTOR = getFloat(IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR, 2.5f);
+
+    /** Size of count of entries (value returned by size() method). */
+    private static final int MAPSIZE_SIZE = 4;
+
+    /** Padding to provide read/write from word beginning for each cell. Change this to 0 if padding is not required. */
+    private static final int CELL_PADDING = 4;
+
+    /** Padding to provide read/write from word beginning. Change this to 0 if padding is not required. */
+    private static final int MAPSIZE_PADDING = 4;
+
+    /** Count of entries offset starting from base address. */
+    private static final int MAPSIZE_OFFSET = 0;
+
+    /** Size of initial/ideal bucket (cell to store value to avoid probing other cells followed). */
+    private static final int IDEAL_BUCKET_SIZE = 4;
+
+    /** Offset of initial/ideal bucket starting from entry base. */
+    private static final int IDEAL_BUCKET_OFFSET = 0;
+
+    /** Group ID size. */
+    private static final int GRP_ID_SIZE = 4;
+
+    /** Group ID offset from entry base. */
+    private static final int GRP_ID_OFFSET = IDEAL_BUCKET_OFFSET + IDEAL_BUCKET_SIZE;
+
+    /** Page ID size. */
+    private static final int PAGE_ID_SIZE = 8;
+
+    /** Page ID offset from entry base. */
+    private static final int PAGE_ID_OFFSET = GRP_ID_OFFSET + GRP_ID_SIZE;
+
+    /** Value size. */
+    private static final int VALUE_SIZE = 8;
+
+    /** Value offset from entry base. */
+    private static final int VALUE_OFFSET = PAGE_ID_OFFSET + PAGE_ID_SIZE;
+
+    /** Version (tag/generation) offset from entry base. */
+    private static final int VERSION_SIZE = 4;
+
+    /** Version (tag/generation) offset from entry base. */
+    private static final int VERSION_OFFSET = VALUE_OFFSET + VALUE_SIZE;
+
+    /** Page ID used for empty bucket. */
+    private static final long EMPTY_PAGE_ID = 0;
+
+    /** Cache Group ID used for empty bucket. */
+    private static final int EMPTY_CACHE_GRP_ID = 0;
+
+    /** Bytes required for storing one entry (cell). */
+    private static final int BYTES_PER_CELL = IDEAL_BUCKET_SIZE
+        + GRP_ID_SIZE + PAGE_ID_SIZE
+        + VALUE_SIZE + VERSION_SIZE
+        + CELL_PADDING;
+
+    /** Number of buckets, indicates range of scan memory, max probe count and maximum map size. */
+    private final int numBuckets;
+
+    /** Base address of map content. */
+    private long baseAddr;
+
+    /**
+     * @param elementsCnt Maximum elements can be stored in map, its maximum size.
+     * @return Estimated memory size required for this map to store the given number of elements.
+     */
+    public static long requiredMemory(long elementsCnt) {
+        float loadFactor = LOAD_FACTOR;
+
+        assert loadFactor != 0;
+
+        return requiredMemoryByBuckets((long)(elementsCnt * loadFactor));
+    }
+
+    /**
+     * @param numBuckets Number of buckets (cells) to store, capacity.
+     * @return required size to allocate, based on number of buckets (cells) to store in map, its capacity.
+     */
+    static long requiredMemoryByBuckets(long numBuckets) {
+        return numBuckets * BYTES_PER_CELL + MAPSIZE_SIZE + MAPSIZE_PADDING;
+    }
+
+    /**
+     * Creates map in preallocated unsafe memory segment.
+     *
+     * @param baseAddr Base buffer address.
+     * @param size Size available for map, number of buckets (cells) to store will be determined accordingly.
+     */
+    public RobinHoodBackwardShiftHashMap(long baseAddr, long size) {
+        this.numBuckets = (int)((size - MAPSIZE_SIZE - MAPSIZE_PADDING) / BYTES_PER_CELL);
+        this.baseAddr = baseAddr;
+
+        GridUnsafe.setMemory(baseAddr, size, (byte)0);
+    }
+
+    /**
+     * @param idx cell index.
+     * @return base cell (bucket) address in buffer.
+     */
+    private long entryBase(int idx) {
+        assert idx >= 0 && idx < numBuckets;
+
+        return baseAddr + MAPSIZE_SIZE + MAPSIZE_PADDING + (long)idx * BYTES_PER_CELL;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long get(int grpId, long pageId, int reqVer, long absent, long outdated) {
+        assert grpId != EMPTY_CACHE_GRP_ID;
+
+        // initial index is also ideal for searhed element
+        int idxInit = U.safeAbs(FullPageId.hashCode(grpId, pageId)) % numBuckets;
+
+        for (int i = 0; i < numBuckets; i++) {
+            int idxCurr = (idxInit + i) % numBuckets;
+
+            final long base = entryBase(idxCurr);
+            final int distanceFromInit = distance(idxCurr, idxInit);
+
+            final int curGrpId = getGrpId(base);
+            final long curPageId = getPageId(base);
+
+            final int dibCurEntry = distance(idxCurr, getIdealBucket(base));
+
+            if (isEmpty(curGrpId, curPageId))
+                return absent;
+            else if (curGrpId == grpId && curPageId == pageId) {
+                //equal value found
+                long actualVer = getVersion(base);
+                boolean freshVal = actualVer >= reqVer;
+
+                return freshVal ? getValue(base) : outdated;
+            }
+            else if (dibCurEntry < distanceFromInit) {
+                //current entry has quite good position, it would be swapped at hypothetical insert of current value
+                return absent;
+            }
+        }
+
+        return absent;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void put(int grpId, long pageId, long val, int ver) {
+        assert grpId != 0;
+
+        // initial index is also ideal for inserted element
+        int idxInit = U.safeAbs(FullPageId.hashCode(grpId, pageId)) % numBuckets;
+
+        int swapCnt = 0;
+
+        int grpIdToInsert = grpId;
+        long pageIdToInsert = pageId;
+        long valToInsert = val;
+        int verToInsert = ver;
+        long idxIdealToInsert = idxInit;
+
+        for (int i = 0; i < numBuckets; i++) {
+            int idxCurr = (idxInit + i) % numBuckets;
+
+            final long base = entryBase(idxCurr);
+            final int dibEntryToInsert = distance(idxCurr, idxInit);
+
+            final int curGrpId = getGrpId(base);
+            final long curPageId = getPageId(base);
+            final int curIdealBucket = getIdealBucket(base);
+            final long curVal = getValue(base);
+            final int curVer = getVersion(base);
+            final int dibCurEntry = distance(idxCurr, curIdealBucket);
+
+            if (isEmpty(curGrpId, curPageId)) {
+                setCellValue(base, idxIdealToInsert, grpIdToInsert, pageIdToInsert, valToInsert, verToInsert);
+
+                setSize(size() + 1);
+
+                return;
+            }
+            else if (curGrpId == grpIdToInsert && curPageId == pageIdToInsert) {
+                if (swapCnt != 0)
+                    throw new IllegalStateException("Swapped " + swapCnt + " times. Entry: " + dumpEntry(idxCurr));
+
+                setValue(base, valToInsert);
+
+                return; //equal value found
+            }
+            else if (dibCurEntry < dibEntryToInsert) {
+                //swapping *toInsert and state in bucket: save cur state to bucket
+                setCellValue(base, idxIdealToInsert, grpIdToInsert, pageIdToInsert, valToInsert, verToInsert);
+
+                idxIdealToInsert = curIdealBucket;
+                pageIdToInsert = curPageId;
+                grpIdToInsert = curGrpId;
+                valToInsert = curVal;
+                verToInsert = curVer;
+
+                swapCnt++;
+            }
+        }
+
+        // no free space left
+        throw new IgniteOutOfMemoryException("No room for a new key");
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean remove(int grpId, long pageId) {
+        assert grpId != EMPTY_CACHE_GRP_ID;
+
+        int idxInit = U.safeAbs(FullPageId.hashCode(grpId, pageId)) % numBuckets;
+
+        int idxEqualValFound = -1;
+        for (int i = 0; i < numBuckets; i++) {
+            int idxCurr = (idxInit + i) % numBuckets;
+
+            final long base = entryBase(idxCurr);
+            final int dibEntryToInsert = distance(idxCurr, idxInit);
+
+            final int curGrpId = getGrpId(base);
+            final long curPageId = getPageId(base);
+            final int curIdealBucket = getIdealBucket(base);
+            final int dibCurEntry = distance(idxCurr, curIdealBucket);
+
+            if (isEmpty(curGrpId, curPageId))
+                return false;
+            else if (curGrpId == grpId && curPageId == pageId) {
+                idxEqualValFound = idxCurr;
+
+                break; //equal value found
+            }
+            else if (dibCurEntry < dibEntryToInsert) {
+                //If our value was present in map we had already found it.
+                return false;
+            }
+        }
+
+        setSize(size() - 1);
+
+        doBackwardShift(idxEqualValFound);
+
+        return true;
+    }
+
+    /**
+     * Runs backward shifts from current index to .
+     *
+     * @param idxRmv removed index.
+     */
+    private void doBackwardShift(int idxRmv) {
+        assert idxRmv >= 0;
+
+        //scanning rest of map to perform backward shifts
+        for (int i = 0; i < numBuckets - 1; i++) {
+            int idxCurr = (idxRmv + i) % numBuckets;
+            int idxNext = (idxRmv + i + 1) % numBuckets;
+
+            long baseCurr = entryBase(idxCurr);
+
+            long baseNext = entryBase(idxNext);
+            final int nextGrpId = getGrpId(baseNext);
+            final long nextPageId = getPageId(baseNext);
+            final int nextIdealBucket = getIdealBucket(baseNext);
+            final int nextEntryVer = getVersion(baseNext);
+
+            if (isEmpty(nextGrpId, nextPageId)
+                || distance(idxNext, nextIdealBucket) == 0) {
+                setEmpty(baseCurr);
+
+                return;
+            }
+            else
+                setCellValue(baseCurr, nextIdealBucket, nextGrpId, nextPageId, getValue(baseNext), nextEntryVer);
+        }
+
+        int lastShiftedIdx = (idxRmv - 1) % numBuckets;
+
+        setEmpty(entryBase(lastShiftedIdx));
+    }
+
+    /** {@inheritDoc} */
+    @Override public ReplaceCandidate getNearestAt(final int idxStart) {
+        for (int i = 0; i < numBuckets; i++) {
+            int idxCurr = (idxStart + i) % numBuckets;
+
+            if (isEmptyAt(idxCurr))
+                continue;
+
+            long base = entryBase(idxCurr);
+
+            return new ReplaceCandidate(getVersion(base), getValue(base), getFullPageId(base));
+        }
+
+        return null;
+    }
+
+    /**
+     * @param idx Index to test.
+     * @return {@code True} if value is not provided in cell having index.
+     */
+    private boolean isEmptyAt(int idx) {
+        long base = entryBase(idx);
+
+        return isEmpty(getGrpId(base), getPageId(base));
+    }
+
+    /** {@inheritDoc} */
+    @Override public long refresh(int grpId, long pageId, int ver) {
+        assert grpId != EMPTY_CACHE_GRP_ID;
+
+        int idxInit = U.safeAbs(FullPageId.hashCode(grpId, pageId)) % numBuckets;
+
+        for (int i = 0; i < numBuckets; i++) {
+            int idxCurr = (idxInit + i) % numBuckets;
+
+            final long base = entryBase(idxCurr);
+            final int distanceFromInit = distance(idxCurr, idxInit);
+
+            final int curGrpId = getGrpId(base);
+            final long curPageId = getPageId(base);
+            final int curIdealBucket = getIdealBucket(base);
+            final int dibCurEntry = distance(idxCurr, curIdealBucket);
+
+            if (isEmpty(curGrpId, curPageId))
+                break; // break to fail
+            else if (curGrpId == grpId && curPageId == pageId) {
+                //equal value found
+                long actualVer = getVersion(base);
+
+                boolean freshVal = actualVer >= ver;
+
+                if (freshVal) {
+                    throw new IllegalArgumentException("Fresh element found at " +
+                        dumpEntry(idxCurr) + " during search of cell to refresh. " +
+                        "Refresh should be called for existent outdated element. ");
+                }
+
+                setVersion(base, ver);
+
+                return getValue(base);
+            }
+            else if (dibCurEntry < distanceFromInit) {
+                //current entry has quite good position,  it would be swapped at hypothetical insert of current value
+
+                break;
+            }
+        }
+
+        throw new IllegalArgumentException("Element not found group ID: " + grpId + ", page ID: " + pageId +
+            " during search of cell to refresh. Refresh should be called for existent outdated element. ");
+    }
+
+    /** {@inheritDoc} */
+    @Override public GridLongList removeIf(int startIdxToClear, int endIdxToClear, KeyPredicate keyPred) {
+        assert endIdxToClear >= startIdxToClear
+            : "Start and end indexes are not consistent: {" + startIdxToClear + ", " + endIdxToClear + "}";
+
+        int sz = endIdxToClear - startIdxToClear;
+
+        GridLongList list = new GridLongList(sz);
+
+        for (int idx = startIdxToClear; idx < endIdxToClear; idx++) {
+            long base = entryBase(idx);
+
+            int grpId = getGrpId(base);
+            long pageId = getPageId(base);
+
+            if (isEmpty(grpId, pageId))
+                continue; // absent value, no removal required
+
+            if (!keyPred.test(grpId, pageId))
+                continue; // not matched value, no removal required
+
+            long valAt = getValue(base);
+
+            setSize(size() - 1);
+
+            doBackwardShift(idx);
+
+            list.add(valAt);
+
+            idx--; //Need recheck current cell because of backward shift
+        }
+
+        return list;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int capacity() {
+        return numBuckets;
+    }
+
+    /**
+     * @param curr current selected index to store value.
+     * @param baseIdx base or ideal bucket to store entry value to avoid probing.
+     * @return distance between cells, or 0 if cell is ideal.
+     */
+    private int distance(int curr, int baseIdx) {
+        int diff = curr - baseIdx;
+
+        if (diff < 0)
+            return diff + numBuckets;
+
+        return diff;
+    }
+
+    /**
+     * @param grpId Cache group ID.
+     * @param pageId Page ID.
+     * @return {@code true} if group & page id indicates cell has state 'Empty'.
+     */
+    private boolean isEmpty(int grpId, long pageId) {
+        return pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID;
+    }
+
+    /**
+     * Sets cell value to be empty.
+     *
+     * @param addr entry base address.
+     */
+    private void setEmpty(long addr) {
+        setPageId(addr, EMPTY_PAGE_ID);
+        setGrpId(addr, EMPTY_CACHE_GRP_ID);
+        setValue(addr, 0);
+        setIdealBucket(addr, 0);
+        setVersion(addr, 0);
+    }
+
+    /**
+     * @param base Entry base, address in buffer of the entry start.
+     * @param idxIdeal number of ideal bucket (cell) to insert this value.
+     */
+    private void setIdealBucket(long base, long idxIdeal) {
+        assert idxIdeal >= 0 && idxIdeal < numBuckets;
+
+        putLong(base + IDEAL_BUCKET_OFFSET, idxIdeal);
+    }
+
+    /**
+     * @return printable dump with all buckets state.
+     */
+    public String dump() {
+        StringBuilder sb = new StringBuilder();
+
+        for (int idx = 0; idx < numBuckets; idx++)
+            dumpEntry(sb, idx);
+
+        return sb.toString();
+    }
+
+    /**
+     * @param idx index of entry to dump
+     * @return string representation of bucket content.
+     */
+    private String dumpEntry(int idx) {
+        StringBuilder sb = new StringBuilder();
+
+        dumpEntry(sb, idx);
+
+        return sb.toString();
+    }
+
+    /**
+     * @param sb destination string builder to dump entry to.
+     * @param idx bucket index.
+     */
+    private void dumpEntry(StringBuilder sb, int idx) {
+        long base = entryBase(idx);
+        int curGrpId = getGrpId(base);
+        long curPageId = getPageId(base);
+        long curVal = getValue(base);
+        long ver = getVersion(base);
+
+        sb.append("slot [").append(idx).append("]:");
+
+        if (isEmpty(curGrpId, curPageId))
+            sb.append("Empty: ");
+
+        sb.append("i.buc=").append(getIdealBucket(base)).append(",");
+        sb.append("(grp=").append(curGrpId).append(",");
+        sb.append("page=").append(curPageId).append(")");
+        sb.append("->");
+        sb.append("(val=").append(curVal).append(",");
+        sb.append("ver=").append(ver).append(")");
+        sb.append("\n");
+    }
+
+    /**
+     * @param base Entry base, address in buffer of the entry start.
+     * @param idealBucket number of ideal bucket (cell) to insert this value.
+     * @param grpId Entry key. Group ID to be stored in entry.
+     * @param pageId Entry key. Page ID to be stored.
+     * @param val Entry value associated with key.
+     * @param ver Entry version.
+     */
+    private void setCellValue(long base, long idealBucket, int grpId, long pageId, long val, int ver) {
+        setIdealBucket(base, idealBucket);
+        setGrpId(base, grpId);
+        setPageId(base, pageId);
+        setValue(base, val);
+        setVersion(base, ver);
+    }
+
+    /**
+     * @param base address of current cell.
+     * @return number of ideal bucket (cell) to store this value.
+     */
+    private int getIdealBucket(long base) {
+        return getInt(base + IDEAL_BUCKET_OFFSET);
+    }
+
+    /**
+     * @param base Address of current cell.
+     * @return Page ID saved in cell.
+     */
+    private long getPageId(long base) {
+        return getLong(base + PAGE_ID_OFFSET);
+    }
+
+    /**
+     * @param base Address of cell.
+     * @param pageId Page ID to set in current cell.
+     */
+    private void setPageId(long base, long pageId) {
+        putLong(base + PAGE_ID_OFFSET, pageId);
+    }
+
+    /**
+     * @param base Entry base address.
+     * @return Group ID stored in entry.
+     */
+    private int getGrpId(long base) {
+        return getInt(base + GRP_ID_OFFSET);
+    }
+
+    /**
+     * @param base Entry base address.
+     * @param grpId Group ID to be stored in entry.
+     */
+    private void setGrpId(long base, int grpId) {
+        putInt(base + GRP_ID_OFFSET, grpId);
+    }
+
+    /**
+     * @param base Bucket base address.
+     * @return value stored in bucket.
+     */
+    private long getValue(long base) {
+        return getLong(base + VALUE_OFFSET);
+    }
+
+    /**
+     * @param base Bucket base address.
+     * @param val Value to store in bucket.
+     */
+    private void setValue(long base, long val) {
+        putLong(base + VALUE_OFFSET, val);
+    }
+
+    /**
+     * @param base Bucket base address.
+     * @return Entry version associated with bucket.
+     */
+    private int getVersion(long base) {
+        return getInt(base + VERSION_OFFSET);
+    }
+
+    /**
+     * @param base Bucket base address.
+     * @param ver Entry version to set in bucket.
+     */
+    private void setVersion(long base, int ver) {
+        putInt(base + VERSION_OFFSET, ver);
+    }
+
+    /** {@inheritDoc} */
+    @Override public final int size() {
+        return GridUnsafe.getInt(baseAddr + MAPSIZE_OFFSET);
+    }
+
+    /**
+     * Changes collection size.
+     *
+     * @param sz new size to set.
+     */
+    private void setSize(int sz) {
+        putInt(baseAddr + MAPSIZE_OFFSET, sz);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void forEach(BiConsumer<FullPageId, Long> act) {
+        for (int i = 0; i < numBuckets; i++) {
+            if (isEmptyAt(i))
+                continue;
+
+            long base = entryBase(i);
+
+            act.accept(getFullPageId(base), getValue(base));
+        }
+    }
+
+    /**
+     * @param base bucket base address.
+     * @return Key. Full page ID from bucket.
+     */
+    @NotNull private FullPageId getFullPageId(long base) {
+        return new FullPageId(getPageId(base), getGrpId(base));
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
index 316cb8b..76cd5bd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
@@ -110,7 +110,7 @@ public class IgniteMassLoadSandboxTest extends GridCommonAbstractTest {
         DataRegionConfiguration regCfg = new DataRegionConfiguration()
             .setName("dfltMemPlc")
             .setMetricsEnabled(true)
-            .setMaxSize(256L * 1024 * 1024)
+            .setMaxSize(2 * 1024L * 1024 * 1024)
             .setPersistenceEnabled(true);
 
         DataStorageConfiguration dsCfg = new DataStorageConfiguration();

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
index 1f29549..43b27aa 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
@@ -20,32 +20,45 @@ package org.apache.ignite.internal.processors.cache.persistence.pagemem;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Random;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
 import org.apache.ignite.internal.mem.DirectMemoryRegion;
 import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
 import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.processors.cache.persistence.pagemem.FullPageIdTable;
-import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.logger.java.JavaLogger;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
 
 /**
  *
  */
-public class FullPageIdTableTest extends GridCommonAbstractTest {
+public class FullPageIdTableTest  {
+    /** */
+    private static final int CACHE_ID_RANGE = 1;
+
+    /** */
+    private static final int PAGE_ID_RANGE = 3000;
+
     /** */
-    private static final int CACHE_ID_RANGE = 10;
+    private static final int CACHE_ID_RANGE2 = 1024;
 
     /** */
-    private static final int PAGE_ID_RANGE = 1000;
+    private static final int PAGE_ID_RANGE2 = 3000;
+
+    /** Logger. */
+    private JavaLogger log = new JavaLogger();
 
     /**
      * @throws Exception if failed.
      */
+    @Test
     public void testRandomOperations() throws Exception {
-        long mem = FullPageIdTable.requiredMemory(CACHE_ID_RANGE * PAGE_ID_RANGE);
+        int cnt = CACHE_ID_RANGE * PAGE_ID_RANGE;
+
+        long mem = FullPageIdTable.requiredMemory(cnt);
 
-        UnsafeMemoryProvider prov = new UnsafeMemoryProvider(new JavaLogger());
+        UnsafeMemoryProvider prov = new UnsafeMemoryProvider(log);
 
         prov.initialize(new long[] {mem});
 
@@ -58,7 +71,7 @@ public class FullPageIdTableTest extends GridCommonAbstractTest {
 
             Random rnd = new Random(seed);
 
-            FullPageIdTable tbl = new FullPageIdTable(region.address(), region.size(), true);
+            LoadedPagesMap tbl = new FullPageIdTable(region.address(), region.size(), true);
 
             Map<FullPageId, Long> check = new HashMap<>();
 
@@ -77,7 +90,7 @@ public class FullPageIdTableTest extends GridCommonAbstractTest {
                     check.put(fullId, val);
                 }
                 else {
-                    tbl.remove(cacheId, pageId, 0);
+                    tbl.remove(cacheId, pageId);
                     check.remove(fullId);
                 }
 
@@ -93,23 +106,163 @@ public class FullPageIdTableTest extends GridCommonAbstractTest {
     }
 
     /**
+     * @throws Exception if failed.
+     */
+    @Test
+    public void putRemoveScenario() throws Exception {
+        long seed = U.currentTimeMillis();
+
+        doPutRemoveTest(seed, false, 1_000_000);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    @Test
+    public void putRemoveScenarioNewMap() throws Exception {
+        long seed = U.currentTimeMillis();
+        doPutRemoveTest(seed, true, 30_000_000);
+    }
+
+    /**
+     * @param seed random initial value.
+     * @param newMapImpl use RobinHood map
+     * @param iters iterations.
+     */
+    private void doPutRemoveTest(long seed, boolean newMapImpl, int iters) {
+        int elementsCnt = 7000;
+
+        //System.setProperty(IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR, "11");
+        long mem = newMapImpl
+            ? RobinHoodBackwardShiftHashMap.requiredMemory(elementsCnt)
+            : FullPageIdTable.requiredMemory(elementsCnt);
+
+        DirectMemoryProvider prov = new UnsafeMemoryProvider(log);
+
+        prov.initialize(new long[] {mem});
+
+        DirectMemoryRegion region = prov.nextRegion();
+
+        try {
+            info("Seed: " + seed + "L; //");
+
+            Random rnd = new Random(seed);
+
+            LoadedPagesMap tbl =
+                newMapImpl
+                    ? new RobinHoodBackwardShiftHashMap(region.address(), region.size())
+                    : new FullPageIdTable(region.address(), region.size(), true);
+
+            Map<FullPageId, Long> check = new HashMap<>();
+
+            int tag = 0;
+            for (int i = 0; i < iters; i++) {
+                int op = rnd.nextInt(5);
+
+                int cacheId = rnd.nextInt(CACHE_ID_RANGE2) + 1;
+                int pageId = rnd.nextInt(PAGE_ID_RANGE2);
+
+                FullPageId fullId = new FullPageId(pageId, cacheId);
+
+                if (op == 0) {
+                    long val = tbl.get(cacheId, pageId, tag, -1, -2);
+                    if (val == -2)
+                        tbl.refresh(cacheId, pageId, tag);
+                    else {
+                        Long checkVal = check.get(fullId);
+
+                        if (checkVal != null) {
+                            assertEquals("Ret " + val + "Check " + checkVal,
+                                checkVal.longValue(), val);
+                        }
+                    }
+
+                }
+                else if ((op == 1 || op == 2) && (check.size() < elementsCnt)) {
+                    long val = U.safeAbs(rnd.nextLong());
+
+                    tbl.put(cacheId, pageId, val, tag);
+                    check.put(fullId, val);
+                }
+                else if ((op == 3) && check.size() >= elementsCnt * 2 / 3) {
+                    tbl.remove(cacheId, pageId);
+                    check.remove(fullId);
+                }
+                else if (check.size() >= elementsCnt * 2 / 3) {
+                    int idx = rnd.nextInt(tbl.capacity());
+                    ReplaceCandidate ec = tbl.getNearestAt(idx);
+                    if (ec != null) {
+                        FullPageId fullPageId = ec.fullId();
+
+                        tbl.remove(fullPageId.groupId(), fullPageId.pageId());
+
+                        check.remove(fullPageId);
+                    }
+                }
+
+                if (i > 0 && i % 100_000 == 0) {
+                    info("Done: " + i
+                        + " Size: " + check.size()
+                        + " Capacity: " + tbl.capacity() );
+
+                    verifyLinear(tbl, check);
+
+                    tag++;
+                }
+
+                i++;
+            }
+
+            verifyLinear(tbl, check);
+        }
+        finally {
+            long msPassed = U.currentTimeMillis() - seed;
+            System.err.println("Seed used [" + seed + "] duration ["+ msPassed+ "] ms");
+            prov.shutdown();
+        }
+    }
+
+    /**
+     * @param msg Message to print.
+     */
+    protected void info(String msg) {
+        if (log.isInfoEnabled())
+            log.info(msg);
+
+        System.out.println(msg);
+    }
+
+
+    /**
      * @param tbl Table to check.
      * @param check Expected mapping.
      */
-    private void verifyLinear(FullPageIdTable tbl, Map<FullPageId, Long> check) {
-        final Map<FullPageId, Long> collector = new HashMap<>();
+    private void verifyLinear(LoadedPagesMap tbl, Map<FullPageId, Long> check) {
+        final Map<FullPageId, Long> tblSnapshot = new HashMap<>();
 
-        tbl.visitAll(new CI2<FullPageId, Long>() {
-            @Override public void apply(FullPageId fullId, Long val) {
-                if (collector.put(fullId, val) != null)
-                    throw new AssertionError("Duplicate full page ID mapping: " + fullId);
-            }
+        tbl.forEach((fullId, val) -> {
+            if (tblSnapshot.put(fullId, val) != null)
+                throw new AssertionError("Duplicate full page ID mapping: " + fullId);
         });
 
-        assertEquals("Size check failed", check.size(), collector.size());
+        int chkSize = check.size();
+        int foundTblSize = tblSnapshot.size();
 
-        for (Map.Entry<FullPageId, Long> entry : check.entrySet())
-            assertEquals("Mapping comparison failed for key: " + entry.getKey(),
-                entry.getValue(), collector.get(entry.getKey()));
+        HashMap<FullPageId, Long> cp = new HashMap<>(tblSnapshot);
+        check.keySet().forEach(cp::remove);
+
+        assertEquals("Size check failed, check map size " +
+            chkSize + " but found in table " + foundTblSize + " elements," +
+            " table size " + tbl.size() +
+            "\n Difference: " + cp, chkSize, foundTblSize);
+
+        for (Map.Entry<FullPageId, Long> entry : check.entrySet()) {
+            Long valCheck = entry.getValue();
+            Long actual = tblSnapshot.get(entry.getKey());
+
+            if (!valCheck.equals(actual))
+                assertEquals("Mapping comparison failed for key: " + entry.getKey(),
+                    valCheck, actual);
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
new file mode 100644
index 0000000..da59145
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
@@ -0,0 +1,430 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.function.Consumer;
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.util.GridLongList;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+import org.jetbrains.annotations.NotNull;
+import org.junit.Test;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Unit tests of {@link RobinHoodBackwardShiftHashMap} implementation.
+ */
+public class RobinHoodBackwardShiftHashMapTest {
+
+    /**
+     * @param tester map test code
+     * @param cap required map capacity.
+     */
+    private void withMap(Consumer<RobinHoodBackwardShiftHashMap> tester, int cap) {
+        long memSize = RobinHoodBackwardShiftHashMap.requiredMemoryByBuckets(cap);
+        long addr = GridUnsafe.allocateMemory(memSize);
+
+        RobinHoodBackwardShiftHashMap map = new RobinHoodBackwardShiftHashMap(addr, memSize);
+        boolean success = false;
+        try {
+            tester.accept(map);
+
+            success = true;
+        }
+        finally {
+            if (!success)
+                System.err.println(map.dump());
+
+            GridUnsafe.freeMemory(addr);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testSimplestPutGet() throws Exception {
+        int cnt = 100;
+        withMap(map -> {
+                for (int i = 0; i < cnt; i++) {
+                    int grpId = i + 1;
+                    int val = grpId * grpId;
+
+                    assertSizeChanged("Unique put should be successful " + grpId,
+                        map, () -> map.put(grpId, 1, val, 1));
+                    assertEquals(val, map.get(grpId, 1, 0, -1, -2));
+
+                    assertSizeNotChanged("Duplicate put for " + grpId,
+                        map, () -> map.put(grpId, 1, 1, 1));
+                    assertEquals(1, map.get(grpId, 1, 0, -1, -2));
+                }
+
+                assertEquals(cnt, map.size());
+            }
+            , cnt);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @Test(expected = IgniteOutOfMemoryException.class)
+    public void testSimplestOverflow() throws Exception {
+        withMap(map -> {
+                for (int i = 0; i < 10; i++) {
+                    int grpId = i + 1;
+                    int val = grpId * grpId;
+                    assertSizeChanged("Unique put should be successful [" + grpId + "]", map, () -> map.put(grpId, 1, val, 1));
+
+                    assertEquals(val, map.get(grpId, 1, 0, -1, -2));
+
+                    assertSizeNotChanged("Duplicate put for " + grpId, map, () -> map.put(grpId, 1, 1, 1));
+                    assertEquals(1, map.get(grpId, 1, 0, -1, -2));
+                }
+
+                map.put(11, 1, 11, 1);
+            }
+            , 10);
+    }
+
+    /**
+     * @param msg message to dump in case assertion failed.
+     * @param map page map to check.
+     * @param act action during which size is expected to be changed.
+     */
+    private static void assertSizeChanged(String msg, LoadedPagesMap map, Runnable act) {
+        int size = map.size();
+        act.run();
+        int newSize = map.size();
+
+        assertNotEquals(msg, size, newSize);
+    }
+
+    /**
+     * @param msg message to dump in case assertion failed.
+     * @param map page map to check.
+     * @param act action during which size is expected to constant.
+     */
+    private static void assertSizeNotChanged(String msg, LoadedPagesMap map, Runnable act) {
+        int size = map.size();
+        act.run();
+        int newSize = map.size();
+
+        assertEquals(msg, size, newSize);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testPutRemoveOnSamePlaces() throws Exception {
+        withMap(map -> {
+                doAddRemove(map);
+
+                //fill with 1 space left;
+                for (int i = 0; i < 99; i++) {
+                    int grpId = i + 1;
+                    int val = grpId * grpId;
+                    assertSizeChanged("Unique put should be successful " + grpId, map,
+                        () -> map.put(grpId, 1, val, 1));
+                }
+
+                doAddRemove(map);
+            }
+            , 100);
+    }
+
+    /**
+     * @param map tested map implementation
+     */
+    private void doAddRemove(LoadedPagesMap map) {
+        for (int i = 0; i < 100; i++) {
+            int grpId = i + 1;
+            int val = grpId * grpId;
+            map.put(grpId, 1, val, 1);
+            assertEquals(val, map.get(grpId, 1, 0, -1, -2));
+
+            assertTrue(map.remove(grpId, 1));
+            assertEquals(-1, map.get(grpId, 1, 0, -1, -2));
+        }
+    }
+
+    /**
+     *
+     */
+    @Test
+    public void testCollisionOnRemove() {
+        Map<FullPageId, Long> ctrl = new LinkedHashMap<>();
+        int cap = 10;
+        FullPageId baseId = new FullPageId(0, 1);
+
+        withMap(map -> {
+            for (int i = 0; i < cap; i++) {
+                int grpId = i + 1;
+                int pageId = findPageIdForCollision(grpId, baseId, cap);
+                ctrl.put(new FullPageId(pageId, grpId), (long)grpId);
+                map.put(grpId, pageId, (long)grpId, 1);
+            }
+            for (FullPageId next : ctrl.keySet()) {
+                assertTrue(map.remove(next.groupId(), next.pageId()));
+            }
+        }, cap);
+    }
+
+    /**
+     * @param grpId Group ID to use
+     * @param id Page to be placed to same bucket with
+     * @param cap map maximum cells.
+     * @return page ID to use in addition to provided {@code grpId} to reach collision.
+     */
+    private int findPageIdForCollision(int grpId, FullPageId id, int cap) {
+        int bucket = U.safeAbs(id.hashCode()) % cap;
+
+        for (int p = 0; p < 1_000_000; p++) {
+            if (U.safeAbs(FullPageId.hashCode(grpId, p)) % cap == bucket)
+                return p;
+        }
+        assertTrue(false);
+        return -1;
+    }
+
+    /**
+     *
+     */
+    @Test
+    public void testRandomOpsPutRemove() {
+        doPutRemoveTest(System.currentTimeMillis());
+    }
+
+    /**
+     * @param seed random seed, use timer to random run.
+     */
+    private void doPutRemoveTest(long seed) {
+        System.setProperty(IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR, "11");
+
+        int elementsCnt = 10_000;
+
+        withMap(tbl -> {
+            Random rnd = new Random(seed);
+
+            Map<FullPageId, Long> check = new HashMap<>();
+
+            int tag = 0;
+            for (int i = 0; i < 20_000_000; i++) {
+                int op = rnd.nextInt(5);
+
+                int cacheId = rnd.nextInt(100) + 1;
+                int pageId = rnd.nextInt(100);
+
+                FullPageId fullId = new FullPageId(pageId, cacheId);
+
+                if (op == 0) {
+                    long val = tbl.get(cacheId, pageId, tag, -1, -2);
+                    if (val == -2)
+                        tbl.refresh(cacheId, pageId, tag);
+                    else {
+                        Long checkVal = check.get(fullId);
+
+                        if (checkVal != null) {
+                            assertEquals("Ret." +
+                                    getPageString(fullId) +
+                                    " tbl: " + val + " Check " + checkVal,
+                                checkVal.longValue(), val);
+                        }
+                    }
+                }
+                else if ((op == 1 || op == 2) && (check.size() < elementsCnt)) {
+                    long val = U.safeAbs(rnd.nextInt(30));
+
+                    check.put(fullId, val);
+
+                    tbl.put(cacheId, pageId, val, tag);
+                }
+                else if ((op == 3) && check.size() >= elementsCnt * 2 / 3) {
+                    tbl.remove(cacheId, pageId);
+
+                    check.remove(fullId);
+                }
+                else if (check.size() >= elementsCnt * 2 / 3) {
+                    ReplaceCandidate ec = tbl.getNearestAt(rnd.nextInt(tbl.capacity()));
+
+                    if (ec != null) {
+                        FullPageId fullPageId = ec.fullId();
+
+                        tbl.remove(fullPageId.groupId(), fullPageId.pageId());
+
+                        check.remove(fullPageId);
+                    }
+                }
+
+                i++;
+            }
+
+        }, elementsCnt);
+    }
+
+    /**
+     * @param fullId page ID.
+     * @return Printable string for page ID.
+     */
+    @NotNull private String getPageString(FullPageId fullId) {
+        return "(grp=" + fullId.groupId() + "," +
+            "page=" + fullId.pageId() + ")";
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testPutAndCantGetOutdatedValue() throws Exception {
+        withMap(map -> {
+            //fill with 1 space left;
+            for (int i = 0; i < 99; i++) {
+                int ver = i;
+                int grpId = ver + 1;
+                int val = grpId * grpId;
+                map.put(grpId, 1, val, ver);
+
+                assertEquals(val, map.get(grpId, 1, ver, -1, -2));
+
+                assertEquals(-2, map.get(grpId, 1, ver + 1, -1, -2));
+            }
+        }, 100);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testPutAndRefreshValue() throws Exception {
+        withMap(map -> {
+            //fill with 1 space left;
+            for (int i = 0; i < 99; i++) {
+                int ver = i;
+                int grpId = ver + 1;
+                int val = grpId * grpId;
+                int pageId = 1;
+                map.put(grpId, pageId, val, ver);
+
+                map.refresh(grpId, pageId, ver + 1);
+
+                assertEquals(val, map.get(grpId, pageId, ver + 1, -1, -2));
+
+            }
+
+            doAddRemove(map);
+        }, 100);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testClearAtWithControlMap3() throws Exception {
+        int cap = 100;
+
+        doRemovalTests(cap, (grpId, pageId) -> {
+            int hc = Integer.hashCode(grpId) + 31 * Long.hashCode(pageId);
+
+            return hc % 3 == 0;
+        });
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testClearAtWithControlMap7() throws Exception {
+        int cap = 100;
+
+        doRemovalTests(cap, (grpId, pageId) -> {
+            int hc = Integer.hashCode(grpId) + 31 * Long.hashCode(pageId);
+
+            return hc % 7 == 0;
+        });
+    }
+
+    /**
+     * @throws Exception  If failed.
+     */
+    @Test
+    public void testClearAllWithControlMap() throws Exception {
+        int cap = 100;
+
+        doRemovalTests(cap, (grpId, pageId) -> true);
+    }
+
+    /**
+     * @param cap capacity of map
+     * @param pred predicate to filter entries to be removed.
+     */
+    private void doRemovalTests(int cap, LoadedPagesMap.KeyPredicate pred) {
+        withMap(map -> {
+            Map<FullPageId, Long> check = new HashMap<>();
+
+            int elems = cap - 1;  //fill with 1 space left;
+            for (int i = 0; i < elems; i++) {
+                int grpId = i + 1;
+                long val = grpId * grpId;
+                int pageId = 1;
+
+                map.put(grpId, pageId, val, i);
+
+                check.put(new FullPageId(pageId, grpId), val);
+            }
+
+            int sz = map.size();
+
+            GridLongList list = map.removeIf(pred);
+            for (int i = 0; i < list.size(); i++) {
+                long val = list.get(i);
+
+                assertTrue(val > 0);
+            }
+
+            assertThat(sz - map.size(), Matchers.is(list.size()));
+
+            Map<FullPageId, Long> res = new HashMap<>();
+
+            map.forEach(res::put);
+
+            assertThat(map.size(), Matchers.is(map.size()));
+
+            for (int i = 0; i < cap; i++) {
+                GridLongList list1 = map.removeIf(i, i + 1, pred);
+
+                assertTrue(list1.isEmpty());  // clear should not be successful.
+            }
+
+            check.keySet().removeIf(entry -> pred.test(entry.groupId(), entry.pageId()));
+
+            MatcherAssert.assertThat(res, Matchers.is(check));
+
+        }, cap);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsUnitTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsUnitTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsUnitTestSuite.java
index 9906429..c022eea 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsUnitTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsUnitTestSuite.java
@@ -16,8 +16,10 @@
  */
 package org.apache.ignite.testsuites;
 
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.FullPageIdTableTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.IgnitePageMemReplaceDelayedWriteUnitTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.IgniteThrottlingUnitTest;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.RobinHoodBackwardShiftHashMapTest;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
 
@@ -27,7 +29,10 @@ import org.junit.runners.Suite;
 @RunWith(Suite.class)
 @Suite.SuiteClasses({
     IgniteThrottlingUnitTest.class,
-    IgnitePageMemReplaceDelayedWriteUnitTest.class
+    IgnitePageMemReplaceDelayedWriteUnitTest.class,
+    IgniteThrottlingUnitTest.class,
+    FullPageIdTableTest.class,
+    RobinHoodBackwardShiftHashMapTest.class
 })
 public class IgnitePdsUnitTestSuite {
 }


[15/50] [abbrv] ignite git commit: IGNITE-7773 Add 'rebalanceClearingPartitionsLeft' metric to .NET - Fixes #3613.

Posted by sb...@apache.org.
IGNITE-7773 Add 'rebalanceClearingPartitionsLeft' metric to .NET - Fixes #3613.

Signed-off-by: Alexey Goncharuk <al...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/447a8a9d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/447a8a9d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/447a8a9d

Branch: refs/heads/ignite-zk
Commit: 447a8a9d0da87128b28d37f4ebaa2e3040e5febb
Parents: ed2435c
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Mon Mar 12 18:16:34 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Mar 12 18:16:34 2018 +0300

----------------------------------------------------------------------
 .../internal/processors/platform/cache/PlatformCache.java   | 1 +
 .../dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs        | 9 +++++++++
 .../Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs       | 7 +++++++
 3 files changed, 17 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/447a8a9d/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
index 86774c1..19dac83 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
@@ -1482,6 +1482,7 @@ public class PlatformCache extends PlatformAbstractTarget {
         writer.writeLong(metrics.getHeapEntriesCount());
         writer.writeLong(metrics.getEstimatedRebalancingFinishTime());
         writer.writeLong(metrics.getRebalancingStartTime());
+        writer.writeLong(metrics.getRebalanceClearingPartitionsLeft());
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/447a8a9d/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
index 43a54c7..a328cf5 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
@@ -637,5 +637,14 @@ namespace Apache.Ignite.Core.Cache
         /// Rebalancing start time.
         /// </returns>
         long RebalancingStartTime { get; }
+
+        /// <summary>
+        /// Gets number of partitions.
+        /// need to be cleared before actual rebalance start.
+        /// </summary>
+        /// <returns>
+        /// Number of clearing partitions for rebalance.
+        /// </returns>
+        long RebalanceClearingPartitionsLeft { get; }
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/447a8a9d/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
index 1a3f0dc..fbc5d4c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
@@ -241,6 +241,9 @@ namespace Apache.Ignite.Core.Impl.Cache
         /** */
         private readonly long _rebalancingStartTime;
 
+        /** */
+        private readonly long _rebalancingClearingPartitionsLeft;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="CacheMetricsImpl"/> class.
         /// </summary>
@@ -319,6 +322,7 @@ namespace Apache.Ignite.Core.Impl.Cache
             _heapEntriesCount = reader.ReadLong();
             _estimatedRebalancingFinishTime = reader.ReadLong();
             _rebalancingStartTime = reader.ReadLong();
+            _rebalancingClearingPartitionsLeft = reader.ReadLong();
         }
 
         /** <inheritDoc /> */
@@ -536,5 +540,8 @@ namespace Apache.Ignite.Core.Impl.Cache
 
         /** <inheritDoc /> */
         public long RebalancingStartTime { get { return _rebalancingStartTime; } }
+
+        /** <inheritDoc /> */
+        public long RebalanceClearingPartitionsLeft { get { return _rebalancingClearingPartitionsLeft; } }
     }
 }
\ No newline at end of file


[36/50] [abbrv] ignite git commit: IGNITE-7965 Robin-hood hashing may fail with negative index in case backward shift finished with full table scan - Fixes #3640.

Posted by sb...@apache.org.
IGNITE-7965 Robin-hood hashing may fail with negative index in case backward shift finished with full table scan - Fixes #3640.

Signed-off-by: Alexey Goncharuk <al...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/96d12d84
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/96d12d84
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/96d12d84

Branch: refs/heads/ignite-zk
Commit: 96d12d84bf1769418af4ff6c0324d8302a067535
Parents: 9fd6f4a
Author: dpavlov <dp...@gridgain.com>
Authored: Thu Mar 15 18:08:04 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Mar 15 18:08:04 2018 +0300

----------------------------------------------------------------------
 .../pagemem/RobinHoodBackwardShiftHashMap.java          |  5 ++++-
 .../pagemem/RobinHoodBackwardShiftHashMapTest.java      | 12 ++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/96d12d84/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
index fd4e0e6..e20673f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
@@ -147,7 +147,7 @@ public class RobinHoodBackwardShiftHashMap implements LoadedPagesMap {
      * @return base cell (bucket) address in buffer.
      */
     private long entryBase(int idx) {
-        assert idx >= 0 && idx < numBuckets;
+        assert idx >= 0 && idx < numBuckets : "idx=" + idx + ", numBuckets=" + numBuckets;
 
         return baseAddr + MAPSIZE_SIZE + MAPSIZE_PADDING + (long)idx * BYTES_PER_CELL;
     }
@@ -320,6 +320,9 @@ public class RobinHoodBackwardShiftHashMap implements LoadedPagesMap {
 
         int lastShiftedIdx = (idxRmv - 1) % numBuckets;
 
+        if (lastShiftedIdx < 0)
+            lastShiftedIdx += numBuckets;
+
         setEmpty(entryBase(lastShiftedIdx));
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/96d12d84/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
index da59145..e594d06 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
@@ -69,6 +69,18 @@ public class RobinHoodBackwardShiftHashMapTest {
      * @throws Exception If failed.
      */
     @Test
+    public void testShortSize() throws Exception {
+        withMap(map -> {
+            map.put(1, 1, 0, 0);
+            map.put(2, 0, 1, 1);
+            map.remove(1, 1);
+        }, 2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @Test
     public void testSimplestPutGet() throws Exception {
         int cnt = 100;
         withMap(map -> {


[41/50] [abbrv] ignite git commit: IGNITE-7890 Fixed error handing on corrupted storage during node startup

Posted by sb...@apache.org.
IGNITE-7890 Fixed error handing on corrupted storage during node startup


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f623ef10
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f623ef10
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f623ef10

Branch: refs/heads/ignite-zk
Commit: f623ef10bb73a34ede5501eed78be362f3ba8c7d
Parents: 094f6c3
Author: Ilya Lantukh <il...@gridgain.com>
Authored: Fri Mar 16 14:25:13 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Mar 16 14:25:13 2018 +0300

----------------------------------------------------------------------
 .../persistence/GridCacheOffheapManager.java    |   9 ++
 .../persistence/pagemem/PageMemoryImpl.java     |   2 +-
 .../IgnitePdsCorruptedStoreTest.java            | 154 +++++++++++++++++++
 .../ignite/testsuites/IgnitePdsTestSuite2.java  |   3 +
 4 files changed, 167 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/f623ef10/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index d3b73bd..313a763 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -27,6 +27,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.NodeInvalidator;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -1179,6 +1180,14 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
 
                     delegate = delegate0;
                 }
+                catch (Throwable ex) {
+                    U.error(log, "Unhandled exception during page store initialization. All further operations will " +
+                        "be failed and local node will be stopped.", ex);
+
+                    NodeInvalidator.INSTANCE.invalidate(ctx.kernalContext(), ex);
+
+                    throw ex;
+                }
                 finally {
                     latch.countDown();
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/f623ef10/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 9f979f5..5657099 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -897,7 +897,7 @@ public class PageMemoryImpl implements PageMemoryEx {
             ByteBuffer restored = curPage == null ? lastValidPage : curPage;
 
             if (restored == null)
-                throw new AssertionError(String.format(
+                throw new IllegalStateException(String.format(
                     "Page is broken. Can't restore it from WAL. (grpId = %d, pageId = %X).",
                     fullId.groupId(), fullId.pageId()
                 ));

http://git-wip-us.apache.org/repos/asf/ignite/blob/f623ef10/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
new file mode 100644
index 0000000..f49ebf6
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsCorruptedStoreTest extends GridCommonAbstractTest {
+    /** */
+    private static final String CACHE_NAME1 = "cache1";
+
+    /** */
+    private static final String CACHE_NAME2 = "cache2";
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+
+        super.beforeTest();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+        cfg.setConsistentId(igniteInstanceName);
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setMaxSize(100 * 1024 * 1024)
+                    .setPersistenceEnabled(true)
+            );
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        cfg.setCacheConfiguration(cacheConfiguration(CACHE_NAME1), cacheConfiguration(CACHE_NAME2));
+
+        return cfg;
+    }
+
+    /**
+     * @return File or folder in work directory.
+     * @throws IgniteCheckedException If failed to resolve file name.
+     */
+    private File file(String file) throws IgniteCheckedException {
+        return U.resolveWorkDirectory(U.defaultWorkDirectory(), file, false);
+    }
+
+    /**
+     * Create cache configuration.
+     *
+     * @param name Cache name.
+     */
+    private CacheConfiguration cacheConfiguration(String name) {
+        CacheConfiguration ccfg = new CacheConfiguration();
+        ccfg.setName(name);
+        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+        ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+        ccfg.setBackups(2);
+
+        return ccfg;
+    }
+
+    /**
+     * @throws Exception If test failed.
+     */
+    public void testNodeInvalidatedWhenPersistenceIsCorrupted() throws Exception {
+        Ignite ignite = startGrid(0);
+
+        startGrid(1);
+
+        ignite.cluster().active(true);
+
+        awaitPartitionMapExchange();
+
+        IgniteCache<Integer, String> cache1 = ignite.cache(CACHE_NAME1);
+
+        for (int i = 0; i < 100; ++i)
+            cache1.put(i, String.valueOf(i));
+
+        forceCheckpoint();
+
+        cache1.put(2, "test");
+
+        String nodeName = ignite.name().replaceAll("\\.", "_");
+
+        stopAllGrids();
+
+        U.delete(file(String.format("db/%s/cache-%s/part-2.bin", nodeName, CACHE_NAME1)));
+
+        ignite = startGrid(1);
+
+        try {
+            startGrid(0);
+        }
+        catch (IgniteCheckedException ex) {
+            // expected
+
+            throw ex;
+        }
+
+        IgniteCache<Integer, Integer> cache2 = ignite.cache(CACHE_NAME2);
+
+        // check that partition map exchange had been finished
+        cache2.put(1, 1);
+
+        assertEquals(1, ignite.cluster().nodes().size());
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/f623ef10/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 5aca92f..f955b11 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -22,6 +22,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgniteDataStorage
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest2;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCorruptedStoreTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsExchangeDuringCheckpointTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
@@ -132,5 +133,7 @@ public class IgnitePdsTestSuite2 extends TestSuite {
         suite.addTestSuite(WalCompactionTest.class);
 
         suite.addTestSuite(IgniteCheckpointDirtyPagesForLowLoadTest.class);
+
+        suite.addTestSuite(IgnitePdsCorruptedStoreTest.class);
     }
 }


[37/50] [abbrv] ignite git commit: IGNITE-7800: Correction for IgniteSystemProperties.IGNITE_NO_SHUTDOWN_HOOK javadoc. - Fixes #3644.

Posted by sb...@apache.org.
IGNITE-7800: Correction for IgniteSystemProperties.IGNITE_NO_SHUTDOWN_HOOK javadoc. - Fixes #3644.

Signed-off-by: shroman <rs...@yahoo.com>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/682197ba
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/682197ba
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/682197ba

Branch: refs/heads/ignite-zk
Commit: 682197ba952e1a011126071062b870fbbdd2f707
Parents: 96d12d8
Author: shroman <rs...@yahoo.com>
Authored: Fri Mar 16 12:13:10 2018 +0900
Committer: shroman <rs...@yahoo.com>
Committed: Fri Mar 16 12:13:10 2018 +0900

----------------------------------------------------------------------
 .../src/main/java/org/apache/ignite/IgniteSystemProperties.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/682197ba/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 11419ec..cd62e48 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -65,7 +65,7 @@ public final class IgniteSystemProperties {
     /** Defines Ignite installation folder. */
     public static final String IGNITE_HOME = "IGNITE_HOME";
 
-    /** If this system property is set to {@code false} - no shutdown hook will be set. */
+    /** If this system property is set to {@code true} - no shutdown hook will be set. */
     public static final String IGNITE_NO_SHUTDOWN_HOOK = "IGNITE_NO_SHUTDOWN_HOOK";
 
     /**


[34/50] [abbrv] ignite git commit: IGNITE-7958 Optimize on-focus-out directive. 1. Don't parse expression every callback call. 2. Use $evalAsync instead of $timeout.

Posted by sb...@apache.org.
IGNITE-7958 Optimize on-focus-out directive.
    1. Don't parse expression every callback call.
    2. Use $evalAsync instead of $timeout.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/22eda40c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/22eda40c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/22eda40c

Branch: refs/heads/ignite-zk
Commit: 22eda40c8d28ca056094d8d3338e5036e46c80d0
Parents: a69e1ac
Author: Ilya Borisov <kl...@gmail.com>
Authored: Thu Mar 15 10:10:17 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Mar 15 10:10:17 2018 +0700

----------------------------------------------------------------------
 .../frontend/app/directives/on-focus-out.directive.js        | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/22eda40c/modules/web-console/frontend/app/directives/on-focus-out.directive.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/directives/on-focus-out.directive.js b/modules/web-console/frontend/app/directives/on-focus-out.directive.js
index 2e6bfc6..802691f 100644
--- a/modules/web-console/frontend/app/directives/on-focus-out.directive.js
+++ b/modules/web-console/frontend/app/directives/on-focus-out.directive.js
@@ -15,15 +15,15 @@
  * limitations under the License.
  */
 
-export default ['$timeout', '$parse', ($timeout, $parse) => {
+export default ['$parse', ($parse) => {
     return ($scope, $element, $attrs) => {
+        const parsedExpr = $parse($attrs.igniteOnFocusOut);
+
         const handlerCheckFocusOut = (FocusClick) => {
             if ($element.find(FocusClick.target).length)
                 return;
 
-            $parse($attrs.igniteOnFocusOut)($scope);
-
-            $timeout();
+            $scope.$evalAsync(() => parsedExpr($scope));
         };
 
         window.addEventListener('click', handlerCheckFocusOut, true);


[48/50] [abbrv] ignite git commit: Merge remote-tracking branch 'remotes/origin/master' into ignite-zk

Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index e848c37,7522881..663053f
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@@ -118,7 -118,7 +118,10 @@@ public enum GridTopic 
      TOPIC_INTERNAL_DIAGNOSTIC,
  
      /** */
 -    TOPIC_WAL;
++    TOPIC_WAL,
++
++    /** */
 +    TOPIC_METRICS;
  
      /** Enum values. */
      private static final GridTopic[] VALS = values();

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 3b320e5,0f952d9..7ee55f9
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@@ -882,16 -887,11 +889,21 @@@ public class GridIoMessageFactory imple
                  break;
  
              case 129:
+                 msg = new WalStateAckMessage();
+ 
+                 break;
+ 
++            case 129:
 +                msg = new ClusterMetricsUpdateMessage();
 +
 +                break;
 +
 +            case 130:
 +                msg = new ContinuousRoutineStartResultMessage();
 +
 +                break;
 +
-             // [-3..119] [124..128] [-23..-27] [-36..-55]- this
+             // [-3..119] [124..129] [-23..-27] [-36..-55]- this
              // [120..123] - DR
              // [-4..-22, -30..-35] - SQL
              // [2048..2053] - Snapshots

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 517a57d,e336667..24dfa2a
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@@ -53,17 -54,12 +54,15 @@@ import org.apache.ignite.cluster.Baseli
  import org.apache.ignite.cluster.ClusterMetrics;
  import org.apache.ignite.cluster.ClusterNode;
  import org.apache.ignite.configuration.CacheConfiguration;
 +import org.apache.ignite.configuration.CommunicationFailureResolver;
  import org.apache.ignite.configuration.DataRegionConfiguration;
  import org.apache.ignite.configuration.DataStorageConfiguration;
 +import org.apache.ignite.configuration.DefaultCommunicationFailureResolver;
 +import org.apache.ignite.configuration.IgniteConfiguration;
  import org.apache.ignite.events.DiscoveryEvent;
  import org.apache.ignite.events.Event;
- import org.apache.ignite.internal.ClusterMetricsSnapshot;
  import org.apache.ignite.internal.GridComponent;
  import org.apache.ignite.internal.GridKernalContext;
- import org.apache.ignite.internal.GridNodeOrderComparator;
  import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
  import org.apache.ignite.internal.IgniteInternalFuture;
  import org.apache.ignite.internal.IgniteKernal;

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
index d87eb8a,9402a32..38450df
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
@@@ -159,24 -159,17 +159,25 @@@ final class BinaryMetadataTransport 
          MetadataUpdateResultFuture resFut = new MetadataUpdateResultFuture();
  
          if (log.isDebugEnabled())
-             log.debug("Requesting metadata update for " + metadata.typeId());
+             log.debug("Requesting metadata update for " + metadata.typeId() + "; caller thread is blocked on future "
+                 + resFut);
  
 -        synchronized (this) {
 -            unlabeledFutures.add(resFut);
 +        try {
 +            synchronized (this) {
 +                unlabeledFutures.add(resFut);
  
 -            if (!stopping)
 -                discoMgr.sendCustomEvent(new MetadataUpdateProposedMessage(metadata, ctx.localNodeId()));
 -            else
 -                resFut.onDone(MetadataUpdateResult.createUpdateDisabledResult());
 +                if (!stopping)
 +                    discoMgr.sendCustomEvent(new MetadataUpdateProposedMessage(metadata, ctx.localNodeId()));
 +                else
 +                    resFut.onDone(MetadataUpdateResult.createUpdateDisabledResult());
 +            }
          }
 +        catch (Exception e) {
 +            resFut.onDone(MetadataUpdateResult.createUpdateDisabledResult(), e);
 +        }
 +
 +        if (ctx.clientDisconnected())
 +            onDisconnected();
  
          return resFut;
      }

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 809a8c5,2d9a314..f0a5186
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@@ -55,8 -55,7 +55,9 @@@ import org.apache.ignite.cluster.Cluste
  import org.apache.ignite.configuration.AddressResolver;
  import org.apache.ignite.configuration.IgniteConfiguration;
  import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 +import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 +import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
+ import org.apache.ignite.internal.IgniteKernal;
  import org.apache.ignite.internal.util.tostring.GridToStringExclude;
  import org.apache.ignite.internal.util.typedef.F;
  import org.apache.ignite.internal.util.typedef.X;

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectApiExceptionTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
index 3fec03f,df93ae4..299dbf4
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
@@@ -113,11 -112,9 +112,11 @@@ public abstract class GridCacheAbstract
       *
       * @throws IgniteCheckedException If failed.
       */
-     void initStoreStrategy() throws IgniteCheckedException {
+     protected void initStoreStrategy() throws IgniteCheckedException {
          if (storeStgy == null)
              storeStgy = isMultiJvm() ? new H2CacheStoreStrategy() : new MapCacheStoreStrategy();
 +        else if (isMultiJvm() && !(storeStgy instanceof H2CacheStoreStrategy))
 +            storeStgy = new H2CacheStoreStrategy();
      }
  
      /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index f5856d5,2337329..92577ad
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@@ -34,10 -35,9 +35,10 @@@ import org.apache.ignite.configuration.
  import org.apache.ignite.configuration.IgniteConfiguration;
  import org.apache.ignite.configuration.WALMode;
  import org.apache.ignite.internal.IgniteClientReconnectAbstractTest;
+ import org.apache.ignite.internal.IgniteEx;
  import org.apache.ignite.internal.IgniteInternalFuture;
- import org.apache.ignite.internal.IgniteKernal;
  import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 +import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
  import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
  import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
  import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteClientDataStructuresAbstractTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 8eef43c,0704dbd..e456047
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@@ -52,9 -52,9 +52,10 @@@ import org.apache.ignite.configuration.
  import org.apache.ignite.configuration.DataStorageConfiguration;
  import org.apache.ignite.configuration.IgniteConfiguration;
  import org.apache.ignite.events.DiscoveryEvent;
 +import org.apache.ignite.internal.DiscoverySpiTestListener;
  import org.apache.ignite.internal.GridKernalContext;
- import org.apache.ignite.internal.GridNodeOrderComparator;
+ import org.apache.ignite.internal.cluster.NodeOrderComparator;
+ import org.apache.ignite.internal.cluster.NodeOrderLegacyComparator;
  import org.apache.ignite.internal.IgniteInternalFuture;
  import org.apache.ignite.internal.IgniteKernal;
  import org.apache.ignite.internal.IgniteNodeAttributes;

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/yardstick/pom-standalone.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/yardstick/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/zookeeper/pom.xml
----------------------------------------------------------------------


[18/50] [abbrv] ignite git commit: IGNITE-7531: Data load benchmarks. This closes #3571.

Posted by sb...@apache.org.
IGNITE-7531: Data load benchmarks. This closes #3571.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ac3d5450
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ac3d5450
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ac3d5450

Branch: refs/heads/ignite-zk
Commit: ac3d545045037446014b66108364bbef39362847
Parents: 1142a36
Author: Pavel Kuznetsov <pa...@gmail.com>
Authored: Tue Mar 13 15:46:36 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Tue Mar 13 15:46:36 2018 +0300

----------------------------------------------------------------------
 .../ignite-localhost-persistence-config.xml     |  71 +++++++
 .../benchmark-jdbc-thin-inmemory.properties     | 108 +++++++++++
 .../benchmark-jdbc-thin-persistence.properties  | 113 +++++++++++
 .../yardstick/IgniteBenchmarkArguments.java     |   9 +-
 .../yardstick/jdbc/AbstractJdbcBenchmark.java   |  23 ++-
 .../upload/AbstractNativeBenchmark.java         | 114 +++++++++++
 .../upload/AbstractUploadBenchmark.java         | 187 +++++++++++++++++++
 .../upload/BatchedInsertBenchmark.java          |  65 +++++++
 .../ignite/yardstick/upload/CopyBenchmark.java  | 125 +++++++++++++
 .../yardstick/upload/InsertBenchmark.java       |  52 ++++++
 .../yardstick/upload/NativePutBenchmark.java    |  39 ++++
 .../upload/NativeStreamerBenchmark.java         |  66 +++++++
 .../upload/UploadBenchmarkArguments.java        | 158 ++++++++++++++++
 .../yardstick/upload/model/QueryFactory.java    | 181 ++++++++++++++++++
 .../ignite/yardstick/upload/model/Values10.java |  76 ++++++++
 .../org/yardstickframework/package-info.java    |  25 +++
 .../probes/TotalTimeProbe.java                  | 137 ++++++++++++++
 17 files changed, 1543 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/config/ignite-localhost-persistence-config.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-localhost-persistence-config.xml b/modules/yardstick/config/ignite-localhost-persistence-config.xml
new file mode 100644
index 0000000..2e9d9ff
--- /dev/null
+++ b/modules/yardstick/config/ignite-localhost-persistence-config.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~      http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!--
+    Ignite Spring configuration file to startup grid.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+    <import resource="ignite-base-config.xml"/>
+
+    <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration" parent="base-ignite.cfg">
+        <property name="localHost" value="127.0.0.1"/>
+
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+                        <property name="addresses">
+                            <list>
+                                <value>127.0.0.1:47500</value>
+                                <value>127.0.0.1:47501</value>
+                                <value>127.0.0.1:47502</value>
+                                <value>127.0.0.1:47503</value>
+                                <value>127.0.0.1:47504</value>
+                                <value>127.0.0.1:47505</value>
+                                <value>127.0.0.1:47506</value>
+                                <value>127.0.0.1:47507</value>
+                                <value>127.0.0.1:47508</value>
+                                <value>127.0.0.1:47509</value>
+                            </list>
+                        </property>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+
+        <property name="dataStorageConfiguration" >
+            <bean class="org.apache.ignite.configuration.DataStorageConfiguration">
+                <property name="defaultDataRegionConfiguration">
+                    <bean class="org.apache.ignite.configuration.DataRegionConfiguration">
+                        <property name="persistenceEnabled" value="true"/>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+
+        <property name="communicationSpi">
+            <bean class="org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi">
+                <property name="sharedMemoryPort" value="-1"/>
+            </bean>
+        </property>
+    </bean>
+</beans>

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties b/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties
new file mode 100644
index 0000000..cacf726
--- /dev/null
+++ b/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Benchmarks for data upload in inmemory mode (persistence disabled).
+#
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
+-XX:+PrintGCDateStamps \
+"
+
+#Ignite version
+ver="RELEASE-"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
+BENCHMARK_DEFAULT_PROBES=TotalTimeProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+# Flag which indicates to restart the servers before every benchmark execution.
+RESTART_SERVERS=true
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# The benchmark is applicable only for 2 servers (the second server is started in client mode) and 1 driver.
+SERVER_HOSTS=localhost,localhost
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Backups count.
+b=1
+
+# Warmup.
+w=0
+
+# Threads count.
+t=1
+
+# Sync mode.
+sm=FULL_SYNC
+
+
+# Run configuration which contains all benchmarks.
+# Note that each benchmark is set to run only one time, warmup parameter is set to 0 due to custom warmup operation.
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn NativePutBenchmark -sn IgniteNode -ds ${ver}sql-upload-native-inmemory-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn NativeStreamerBenchmark -sn IgniteNode -ds ${ver}sql-upload-native-inmemory-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+  --streamer-local-batch-size 1000, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn CopyBenchmark -sn IgniteNode -ds ${ver}sql-upload-inmemory-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn InsertBenchmark -sn IgniteNode -ds ${ver}sql-upload-inmemory-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn BatchedInsertBenchmark -sn IgniteNode -ds ${ver}sql-upload-inmemory-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+  --upload-jdbc-batch-size 1000 \
+"

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties b/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties
new file mode 100644
index 0000000..b72e9b7
--- /dev/null
+++ b/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties
@@ -0,0 +1,113 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Benchmarks for data upload with enabled persistence.
+#
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
+-XX:+PrintGCDateStamps \
+"
+
+#Ignite version
+ver="RELEASE-"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
+BENCHMARK_DEFAULT_PROBES=TotalTimeProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+# Flag which indicates to restart the servers before every benchmark execution.
+RESTART_SERVERS=true
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# The benchmark is applicable only for 2 servers (the second server is started in client mode) and 1 driver.
+SERVER_HOSTS=localhost,localhost
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Backups count.
+b=1
+
+# Warmup.
+w=0
+
+# Threads count.
+t=1
+
+# Sync mode.
+sm=FULL_SYNC
+
+# Whither or not turn off WAL just for upload period.
+switchWal=false
+
+# Run configuration which contains all benchmarks.
+# Note that each benchmark is set to run only one time, warmup parameter is set to 0 due to custom warmup operation.
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn NativePutBenchmark -sn IgniteNode -ds ${ver}sql-upload-native-persistence-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn NativeStreamerBenchmark -sn IgniteNode -ds ${ver}sql-upload-native-persistence-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+  --streamer-local-batch-size 1000, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn CopyBenchmark -sn IgniteNode -ds ${ver}sql-upload-persistence-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+  --disable-wal ${switchWal}, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn InsertBenchmark -sn IgniteNode -ds ${ver}sql-upload-persistence-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+  --disable-wal ${switchWal}, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+  -jdbc jdbc:ignite:thin://auto.find/ \
+  --threads ${t} \
+  --syncMode ${sm} -dn BatchedInsertBenchmark -sn IgniteNode -ds ${ver}sql-upload-persistence-jdbc-thin-r1-${b}-backup \
+  --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+  --upload-jdbc-batch-size 1000 \
+  --disable-wal ${switchWal} \
+"

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
index 8c453ad..9789aea 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.yardstick;
 
 import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParametersDelegate;
+import java.util.Collections;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.configuration.DataStorageConfiguration;
@@ -29,6 +31,7 @@ import org.apache.ignite.transactions.TransactionIsolation;
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.ignite.yardstick.cache.IgniteStreamerBenchmark;
+import org.apache.ignite.yardstick.upload.UploadBenchmarkArguments;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -262,6 +265,10 @@ public class IgniteBenchmarkArguments {
     @GridToStringInclude
     private int clientNodesAfterId = -1;
 
+    @ParametersDelegate
+    @GridToStringInclude
+    public UploadBenchmarkArguments upload = new UploadBenchmarkArguments();
+
     /**
      * @return {@code True} if need set {@link DataStorageConfiguration}.
      */
@@ -384,7 +391,7 @@ public class IgniteBenchmarkArguments {
     /**
      * @return {@code True} if flag for native benchmarking is set.
      */
-    public boolean isNative(){
+    public boolean isNative() {
         return ntv;
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
index 9ef8a77..6ab024f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
@@ -37,18 +37,18 @@ import static org.yardstickframework.BenchmarkUtils.println;
 /**
  * JDBC benchmark that performs query operations.
  */
-abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
+public abstract class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
     /** All {@link Connection}s associated with threads. */
     private final List<Connection> threadConnections = new ArrayList<>();
 
     /** JDBC URL. */
-    private String url;
+    protected String url;
 
     /** Each connection is also a transaction, so we better pin them to threads. */
     protected ThreadLocal<Connection> conn = new ThreadLocal<Connection>() {
         @Override protected Connection initialValue() {
             try {
-                Connection conn = connection();
+                Connection conn = connection(url);
 
                 synchronized (threadConnections) {
                     threadConnections.add(conn);
@@ -66,6 +66,9 @@ abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
     @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
         super.setUp(cfg);
 
+        // activate cluster if it is not auto activated
+        ignite().cluster().active(true);
+
         if (url == null) {
             if (args.jdbcUrl().startsWith(JdbcThinUtils.URL_PREFIX)) {
                 String addr = findThinAddress();
@@ -77,12 +80,22 @@ abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
 
         println("Using jdbc url:" + url);
 
-        fillData(cfg, (IgniteEx)ignite(), args.range());
+        setupData();
 
         ignite().close();
     }
 
     /**
+     * Sets up test data
+     *
+     * Gets executed before local Ignite node is closed
+     * @throws Exception On error.
+     */
+    protected void setupData() throws Exception {
+        fillData(cfg, (IgniteEx)ignite(), args.range());
+    }
+
+    /**
      * Find address of client node, that thin driver should use.
      *
      * @return Address for thin driver.
@@ -128,7 +141,7 @@ abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
      * @return JDBC connection.
      * @throws SQLException On error.
      */
-    private Connection connection() throws SQLException {
+    protected final Connection connection(String url) throws SQLException {
         println("JDBC connect to: " + url);
 
         Connection conn = DriverManager.getConnection(url);

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
new file mode 100644
index 0000000..de1ad28
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.util.Map;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
+import org.apache.ignite.yardstick.upload.model.Values10;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Base class for benchmarks that perform upload using java api.
+ */
+public abstract class AbstractNativeBenchmark extends IgniteAbstractBenchmark {
+    /** Number of entries to be uploaded during warmup. */
+    private long insertRowsCnt;
+
+    /** Name of the {@link #cache} */
+    private String cacheName;
+
+    /** Cache method {@link test(Map)} uploads data to */
+    private IgniteCache<Long, Values10> cache;
+
+    /**
+     * Sets up benchmark: performs warmup on one cache and creates another for {@link #test(Map)} method.
+     *
+     * @param cfg Benchmark configuration.
+     * @throws Exception - on error.
+     */
+    @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        cacheName = this.getClass().getSimpleName();
+
+        insertRowsCnt = args.upload.uploadRowsCnt();
+
+        // Number of entries to be uploaded during test().
+        long warmupRowsCnt = args.upload.warmupRowsCnt();
+
+        // warmup
+        BenchmarkUtils.println(cfg, "Starting custom warmup.");
+        String warmupCacheName = cacheName + "Warmup";
+
+        try (IgniteCache<Long, Values10> warmupCache = ignite().createCache(warmupCacheName)) {
+            upload(warmupCacheName, warmupRowsCnt);
+        }
+        finally {
+            ignite().destroyCache(warmupCacheName);
+        }
+
+        BenchmarkUtils.println(cfg, "Custom warmup finished.");
+
+        // cache for benchmarked action
+        cache = ignite().createCache(cacheName);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void tearDown() throws Exception {
+        try {
+            if (cache == null)
+                throw new IllegalStateException("Cache is null, probably an error during setUp or warmup");
+
+            long size = cache.sizeLong();
+
+            if (size != insertRowsCnt) {
+                String msg = "Incorrect cache size: [actual=" + size + ", expected=" + insertRowsCnt +"].";
+
+                BenchmarkUtils.println(cfg, "TearDown: " + msg);
+
+                throw new RuntimeException(msg);
+            }
+
+            cache.close();
+
+            ignite().destroyCache(cacheName);
+
+        }
+        catch (IgniteException ex) {
+            BenchmarkUtils.println(cfg, "Could not close or destroy cache: " + ex);
+
+            throw ex;
+        }
+        finally {
+            super.tearDown();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        upload(cacheName, insertRowsCnt);
+
+        return true;
+    }
+
+    /** Uploads {@param insertsCnt} to cache with name {@param cacheName} using java api. */
+    protected abstract void upload(String cacheName, long insertsCnt);
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
new file mode 100644
index 0000000..81a93e2
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.yardstick.jdbc.AbstractJdbcBenchmark;
+import org.apache.ignite.yardstick.upload.model.QueryFactory;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Base class for upload benchmarks.
+ * Designed to run test method one single time.
+ * Introduces custom warmup operation.
+ */
+public abstract class AbstractUploadBenchmark extends AbstractJdbcBenchmark {
+    /** Total inserts size. */
+    long insertRowsCnt;
+
+    /** Rows count to be inserted and deleted during warmup */
+    long warmupRowsCnt;
+
+    /** Factory that hides all the test data details. */
+    protected QueryFactory queries = new QueryFactory();
+
+    /** {@inheritDoc} */
+    @Override public final void setUp(BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        insertRowsCnt = args.upload.uploadRowsCnt();
+        warmupRowsCnt = args.upload.warmupRowsCnt();
+
+        init();
+
+        // Perform warmup keeping in mind wal optimization.
+        BenchmarkUtils.println(this.cfg, "Starting custom warmup");
+
+        if (args.upload.disableWal())
+            executeUpdate(QueryFactory.TURN_OFF_WAL);
+
+        try (Connection warmupConn = uploadConnection()) {
+            warmup(warmupConn);
+        }
+
+        if (args.upload.disableWal())
+            executeUpdate(QueryFactory.TURN_ON_WAL);
+
+        BenchmarkUtils.println(this.cfg, "Custom warmup finished");
+
+        dropAndCreate();
+    }
+
+    /**
+     * Inits benchmark fields.
+     */
+    protected void init() {
+        // No-op.
+    }
+
+    /**
+     * Method to warm up Benchmark server. <br/>
+     * In upload benchmarks we need warmup action
+     * and real test action to be separated.
+     */
+    protected abstract void warmup(Connection warmupConn) throws Exception;
+
+    /**
+     * Creates empty table.
+     */
+    @Override protected void setupData() throws Exception{
+        dropAndCreate();
+    }
+
+    /**
+     * Uploads data using this special connection, that may have additional
+     * url parameters, such as {@code streaming=true}.
+     */
+    protected abstract void upload(Connection uploadConn) throws Exception;
+
+    /** {@inheritDoc} */
+    @Override public final boolean test(Map<Object, Object> ctx) throws Exception {
+        if (args.upload.disableWal())
+            executeUpdate(QueryFactory.TURN_OFF_WAL);
+
+        try (Connection uploadConn = uploadConnection()) {
+            upload(uploadConn);
+        }
+
+        if (args.upload.disableWal())
+            executeUpdate(QueryFactory.TURN_ON_WAL);
+
+        return true;
+    }
+
+    /**
+     * Drops and re-creates test table.
+     */
+    private final void dropAndCreate() throws SQLException {
+        executeUpdate(QueryFactory.DROP_TABLE_IF_EXISTS);
+        executeUpdate(queries.createTable());
+    }
+
+    /**
+     * Retrieves records count in the test table.
+     */
+    public long count() throws SQLException {
+        try(PreparedStatement cnt = conn.get().prepareStatement(QueryFactory.COUNT)){
+            try (ResultSet rs = cnt.executeQuery()) {
+                rs.next();
+
+                return rs.getLong(1);
+            }
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void tearDown() throws Exception {
+        BenchmarkUtils.println(cfg, "Tearing down");
+
+        try {
+            long cnt = count();
+
+            if (cnt != insertRowsCnt) {
+                String msg = "Rows count is incorrect: [actual=" + cnt + ", expected=" + insertRowsCnt + "]";
+
+                BenchmarkUtils.println(cfg, "TearDown: " + msg);
+
+                throw new RuntimeException(msg);
+            }
+
+            BenchmarkUtils.println(cfg, "Test table contains " + cnt + " rows.");
+        }
+        finally {
+            super.tearDown();
+        }
+
+        BenchmarkUtils.println(cfg, "TearDown successfully finished.");
+    }
+
+    /**
+     * Facility method for executing update queries.
+     */
+    private int executeUpdate(String updQry) throws SQLException {
+        try(PreparedStatement update = conn.get().prepareStatement(updQry)){
+            return update.executeUpdate();
+        }
+    }
+
+    /**
+     *  Creates new connection only for upload purpose.
+     *  This connection is special, since it may have additional jdbc url parameters.
+     */
+    private Connection uploadConnection() throws SQLException {
+        String urlParams = "";
+
+        // We can't just pass entire params string, due to yardstick, which relies on bash,
+        // has some troubles with escaping ampersand character.
+        List<String> rawParams = args.upload.uploadJdbcParams();
+
+        if (!rawParams.isEmpty()) {
+            String kvList = String.join("&", rawParams);
+            urlParams = "?" + kvList;
+        }
+
+        return connection(url + urlParams);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
new file mode 100644
index 0000000..a460e20
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Measures total time of data upload using batched insert.
+ * Supports streaming.
+ */
+public class BatchedInsertBenchmark extends AbstractUploadBenchmark {
+    /** Number of inserts in batch */
+    private long batchSize;
+
+    /** {@inheritDoc} */
+    @Override protected void init() {
+        batchSize = args.upload.jdbcBatchSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void warmup(Connection warmupConn) throws SQLException {
+        performBatchUpdate(warmupConn, warmupRowsCnt);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void upload(Connection uploadConn) throws Exception {
+        performBatchUpdate(uploadConn, insertRowsCnt);
+    }
+
+    /**
+     * Actually performs batched inserts, using specified connection.
+     *
+     * @param uploadConn Special connection for upload purposes.
+     * @param rowsCnt Number of rows to insert.
+     */
+    private void performBatchUpdate(Connection uploadConn, long rowsCnt) throws SQLException {
+        try (PreparedStatement insert = uploadConn.prepareStatement(queries.insert())) {
+            for (long id = 1; id <= rowsCnt; id++) {
+                queries.setRandomInsertArgs(insert, id);
+
+                insert.addBatch();
+
+                if (id % batchSize == 0 || id == rowsCnt)
+                    insert.executeBatch();
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
new file mode 100644
index 0000000..f0dcf20
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.io.File;
+import java.io.PrintStream;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Single shot benchmark for the COPY FROM... sql query.
+ * Measures total time of load number of rows from csv file using thin driver.
+ *
+ * Enabling streaming is useless for this benchmark, since copy uses streamer under the hood.
+ */
+public class CopyBenchmark extends AbstractUploadBenchmark {
+    /** csv file for warmup. */
+    private String warmupCsvPath;
+
+    /** csv file for benchmared action. */
+    private String realCsvPath;
+
+    /** packet_size parameter for sql COPY command.*/
+    private Long packetSize;
+
+    /** {@inheritDoc} */
+    @Override protected void init() {
+        super.init();
+
+        packetSize = args.upload.copyPacketSize();
+
+        warmupCsvPath = generateWarmupCsv();
+        realCsvPath = generateRealCsv();
+
+    }
+
+    /**
+     * Generate csv file for copy operation being benchmarked.
+     */
+    private String generateRealCsv() {
+        String prefix = "data-" + insertRowsCnt + "-rows-";
+
+        return generate(prefix, insertRowsCnt);
+    }
+
+    /**
+     * Generate csv file for copy operation being performed during warmup.
+     */
+    private String generateWarmupCsv() {
+        String prefix = "warmup-" + warmupRowsCnt + "-rows-" ;
+
+        return generate(prefix, warmupRowsCnt);
+    }
+
+    /**
+     * Generate csv file with specified prefix and containing specified number
+     * of records (lines).
+     * <br/>
+     * Each record contain incremental (starting 1) id and set of random values.
+     * <br/>
+     * File is created in current directory and is deleted on JVM exits.
+     *
+     * @param filePrefix prefix for csv to be generated.
+     * @param records how many csv lines generate.
+     *
+     * @return absolute path for generated csv file.
+     */
+    private String generate(String filePrefix, long records){
+        try {
+            File workDir = new File(System.getProperty("user.dir"));
+
+            File f = File.createTempFile(filePrefix, ".csv", workDir);
+
+            f.deleteOnExit();
+
+            BenchmarkUtils.println("Generating file: " + f.getAbsolutePath());
+
+            try (PrintStream out = new PrintStream(f)) {
+                for (long id = 1; id <= records; id++) {
+                    String csvLine = queries.randomCsvLine(id);
+                    out.println(csvLine);
+                }
+            }
+
+            long sizeMb = f.length() / (1024L * 1024L);
+
+            BenchmarkUtils.println("File have been generated (" + sizeMb + "MiB). It will be removed on exit");
+
+            return f.getAbsolutePath();
+        }
+        catch (Exception ex){
+            throw new RuntimeException("Couldn't generate CSV file, terminating", ex);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void warmup(Connection warmupConn) throws Exception {
+        try (PreparedStatement fromCsv = warmupConn.prepareStatement(queries.copyFrom(warmupCsvPath, packetSize))) {
+            fromCsv.executeUpdate();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void upload(Connection uploadConn) throws Exception {
+        try (PreparedStatement fromCsv = uploadConn.prepareStatement(queries.copyFrom(realCsvPath, packetSize))) {
+            fromCsv.executeUpdate();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
new file mode 100644
index 0000000..ecf5412
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Measures total time of upload data using sequence of single inserts;
+ * Supports streaming.
+ */
+public class InsertBenchmark extends AbstractUploadBenchmark {
+    /** {@inheritDoc} */
+    @Override public void warmup(Connection warmupConn) throws SQLException {
+        try (PreparedStatement insert = warmupConn.prepareStatement(queries.insert())) {
+            for (long id = 1; id <= warmupRowsCnt; id++) {
+                queries.setRandomInsertArgs(insert, id);
+
+                insert.executeUpdate();
+            }
+        }
+    }
+
+    /**
+     * Sequence of single inserts.
+     */
+    @Override public void upload (Connection uploadConn) throws Exception {
+        try (PreparedStatement insert = uploadConn.prepareStatement(queries.insert())) {
+            for (long id = 1; id <= insertRowsCnt; id++) {
+                queries.setRandomInsertArgs(insert, id);
+
+                insert.executeUpdate();
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
new file mode 100644
index 0000000..60fbbda
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.yardstick.upload.model.Values10;
+
+/**
+ * Benchmark that inserts single upload of number of entries using {@link IgniteCache#put(Object, Object)}.
+ */
+public class NativePutBenchmark extends AbstractNativeBenchmark {
+    /**
+     * Uploads randomly generated data using simple put.
+     *
+     * @param cacheName - name of the cache.
+     * @param insertsCnt - how many entries should be uploaded.
+     */
+    @Override protected void upload(String cacheName, long insertsCnt) {
+        IgniteCache<Object, Object> c = ignite().cache(cacheName);
+
+        for (long id = 1; id <= insertsCnt; id++)
+            c.put(id, new Values10() );
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
new file mode 100644
index 0000000..e5117bc
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.yardstick.upload.model.Values10;
+
+/**
+ * Benchmark that performs single upload of number of entries using {@link IgniteDataStreamer}.
+ */
+public class NativeStreamerBenchmark extends AbstractNativeBenchmark {
+    /**
+     * Uploads randomly generated entries to specified cache.
+     *
+     * @param cacheName - name of the cache.
+     * @param insertsCnt - how many entries should be uploaded.
+     */
+    @Override protected void upload(String cacheName, long insertsCnt) {
+        try (IgniteDataStreamer<Long, Values10> streamer = ignite().dataStreamer(cacheName)) {
+            if (args.upload.streamerBufSize() != null)
+                streamer.perNodeBufferSize(args.upload.streamerBufSize());
+
+            if (args.upload.streamerNodeParOps() != null)
+                streamer.perNodeParallelOperations(args.upload.streamerNodeParOps());
+
+            int batchSize = args.upload.streamerLocBatchSize();
+
+            // IgniteDataStreamer.addData(Object, Object) has known performance issue,
+            // so we have an option to work it around.
+            if (batchSize == 1) {
+                for (long i = 1; i <= insertsCnt; i++)
+                    streamer.addData(i, new Values10());
+            }
+            else {
+                Map<Long, Values10> buf = new HashMap<>(batchSize);
+
+                for (long i = 1; i <= insertsCnt; i++) {
+                    buf.put(i, new Values10());
+
+                    if (i % batchSize == 0 || i == insertsCnt) {
+                        streamer.addData(buf);
+
+                        buf.clear();
+                    }
+                }
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
new file mode 100644
index 0000000..27a2164
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import com.beust.jcommander.Parameter;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
+import org.apache.ignite.yardstick.IgniteBenchmarkArguments;
+
+/**
+ * Represents command line arguments that are specific to upload benchmarks.
+ *
+ * @see IgniteBenchmarkArguments
+ */
+@SuppressWarnings({"UnusedDeclaration", "FieldCanBeLocal"})
+public class UploadBenchmarkArguments {
+    /** Whither or not temporary disable Write Ahead Log during upload. */
+    @Parameter(names = {"--disable-wal"},
+        arity = 1,
+        description = "Upload benchmark only: " +
+            "turn off Write Ahead Log before data uploading " +
+            "and turn it on again when upload is done.")
+    private boolean disableWal = false;
+
+    /**
+     * Parameters for JDBC connection, that only uploads data.
+     *
+     * We can't just pass entire params string, due to yardstick, which relies on bash,
+     * has some troubles with escaping ampersand character.
+     */
+    @Parameter(names = {"--sql-jdbc-params"},
+        variableArity = true,
+        description = "Upload benchmark only: " +
+            "Additional url parameters (space separated key=value) for special JDBC connection that only uploads data. ")
+    private List<String> uploadJdbcParams = Collections.emptyList();
+
+    @Parameter(names = {"--sql-copy-packet-size"},
+        description = "Upload benchmark only: use custom packet_size (in bytes) for copy command.")
+    private Long copyPacketSize = null;
+
+    @Parameter(names = {"--streamer-node-buf-size"},
+        description = "Native Streamer benchmark only: Set streamer's perNodeBufferSize property")
+    private Integer streamerBufSize = null;
+
+    @Parameter(names = {"--streamer-node-par-ops"},
+        description = "Native Streamer benchmark only: Set streamer's perNodeParallelOperations property")
+    private Integer streamerNodeParOps = null;
+
+    @Parameter(names = {"--streamer-local-batch-size"},
+        description = "Native Streamer benchmark only: collect entries to local map before passing to streamer." +
+            "If set to 1, than entries will be passed directly.")
+    private int streamerLocBatchSize = 1;
+
+    /** How many rows to upload during warmup. */
+    @Parameter(names = {"--upload-warmup-rows"})
+    private long warmupRowsCnt = 3_000_000;
+
+    /** How many rows to upload during real test. */
+    @Parameter(names = {"--upload-rows"})
+    private long uploadRowsCnt = -1;
+
+    /** How many rows to include in each batch ({@link BatchedInsertBenchmark} only). */
+    @Parameter(names = {"--upload-jdbc-batch-size"})
+    private long jdbcBatchSize = -1;
+
+    /**
+     * @return Switch wal.
+     */
+    public boolean disableWal() {
+        return disableWal;
+    }
+
+    /** @return parameters for JDBC url. */
+    public List<String> uploadJdbcParams() {
+        return uploadJdbcParams;
+    }
+
+    /** @return packet_size value for copy command or {@code null} for default value. */
+    @Nullable
+    public Long copyPacketSize() {
+        return copyPacketSize;
+    }
+
+    /**
+     * @return Value for {@link IgniteDataStreamer#perNodeBufferSize(int)}.
+     */
+    @Nullable
+    public Integer streamerBufSize() {
+        return streamerBufSize;
+    }
+
+    /**
+     * @return Value for {@link IgniteDataStreamer#perNodeParallelOperations(int)}.
+     */
+    @Nullable
+    public Integer streamerNodeParOps() {
+        return streamerNodeParOps;
+    }
+
+    /**
+     * How many entries to collect before passing map to {@link IgniteDataStreamer#addData(Map)}.
+     * If set to 1, {@link IgniteDataStreamer#addData(Object, Object)} method will be used.
+     */
+    public int streamerLocBatchSize() {
+        return streamerLocBatchSize;
+    }
+
+    /**
+     * See {@link #warmupRowsCnt}.
+     */
+    public long warmupRowsCnt() {
+        return warmupRowsCnt;
+    }
+
+    /**
+     * See {@link #uploadRowsCnt}.
+     */
+    public long uploadRowsCnt() {
+        if (uploadRowsCnt < 0)
+            throw new IllegalStateException("Upload rows count is not specified. Check arguments.");
+
+        return uploadRowsCnt;
+    }
+
+    /**
+     * See {@link #jdbcBatchSize}.
+     */
+    public long jdbcBatchSize() {
+        if (jdbcBatchSize < 0)
+            throw new IllegalStateException("JDBC batch size is not specified. Check arguments.");
+
+        return jdbcBatchSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return GridToStringBuilder.toString(UploadBenchmarkArguments.class, this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
new file mode 100644
index 0000000..e14a091
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload.model;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.concurrent.ThreadLocalRandom;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Factory that hides all test data details:
+ * what query to use to create table
+ * or what random arguments to set in prepared statement.
+ */
+public class QueryFactory {
+    /** Query to drop table if it exists. */
+    public static final String DROP_TABLE_IF_EXISTS = "DROP TABLE IF EXISTS test_upload;";
+
+    /** Query to count table size. */
+    public static final String COUNT = "SELECT COUNT(id) FROM test_upload;";
+
+    /** Turns off Write Ahead Log. */
+    public static final String TURN_OFF_WAL = "ALTER TABLE test_upload NOLOGGING";
+
+    /** Turns on Write Ahead Log. */
+    public static final String TURN_ON_WAL = "ALTER TABLE test_upload LOGGING";
+
+    /** Number of "values" fields in the test table (any field except primary key). */
+    private int valFieldsCnt = 10;
+
+    /** Create table with long primary key and number of long and varchar fields */
+    private String createTable = newCreateTableQuery();
+
+    /** Parametrised query to insert new row. */
+    private String insert = newInsertQuery();
+
+    /**
+     * See {@link #createTable}.
+     */
+    private String newCreateTableQuery() {
+        StringBuilder create = new StringBuilder("CREATE TABLE test_upload (id LONG PRIMARY KEY");
+
+        for (int vi = 1; vi <= valFieldsCnt; vi++) {
+            create.append(", val_").append(vi);
+
+            if (vi % 2 == 1)
+                create.append(" VARCHAR(255)");
+            else
+                create.append(" LONG");
+
+        }
+
+        create.append(");");
+
+        return create.toString();
+    }
+
+    /**
+     * See {@link #insert}.
+     */
+    private String newInsertQuery() {
+        StringBuilder insert = new StringBuilder("INSERT INTO test_upload VALUES (?");
+        for (int vi = 1; vi <= valFieldsCnt; vi++)
+            insert.append(", ?");
+
+        insert.append(");");
+        return insert.toString();
+    }
+
+    /**
+     * See {@link #createTable}.
+     */
+    public String createTable() {
+        return createTable;
+    }
+
+    /**
+     * See {@link #insert}.
+     */
+    public String insert() {
+        return insert;
+    }
+
+    /**
+     * @param csvFilePath path to csv file.
+     * @param packetSize if not null, add packet_size query option.
+     * @return sql query that inserts data from specified csv file.
+     */
+    public String copyFrom(String csvFilePath, @Nullable Long packetSize) {
+        String pSizeExpr = "";
+
+        if (packetSize != null)
+            pSizeExpr = " packet_size " + packetSize;
+
+        return "COPY FROM \"" + csvFilePath + "\" " +
+            "INTO test_upload " + attributes() + " " +
+            "FORMAT CSV" + pSizeExpr + ";";
+    }
+
+    /**
+     * Creates string - comma-separated attributes of test table, surrounded with braces
+     * Is used as a part of sql statement.
+     *
+     * @return attributes list of test table as part of sql statement.
+     */
+    private String attributes() {
+        StringBuilder attrs = new StringBuilder("(id");
+
+        for (int vi = 1; vi <= valFieldsCnt; vi++)
+            attrs.append(", val_").append(vi);
+
+        attrs.append(')');
+
+        return attrs.toString();
+    }
+
+    /**
+     * Fills specified prepared statement with random values and specified id (primary key).
+     *
+     * @param stmt prepared statement, built from {@link #insert} query.
+     * @param id id in the test table.
+     * @throws SQLException if statement is not correct.
+     */
+    public void setRandomInsertArgs(PreparedStatement stmt, long id) throws SQLException {
+        stmt.setLong(1, id);
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        for (int vi = 1; vi <= valFieldsCnt; vi++) {
+            // vi is value index (among all values), but we also have "id" which is primary key
+            // so index in query is value index shifted by 1
+            int qryIdx = vi + 1;
+
+            long nextVal = rnd.nextLong();
+
+            if (vi % 2 == 1)
+                stmt.setLong(qryIdx, nextVal);
+            else
+                stmt.setString(qryIdx, String.valueOf(nextVal));
+        }
+    }
+
+    /**
+     * Generates CSV line containing specified id and random values.
+     * This line corresponds 1 row of the test table,
+     * which will be inserted in the end.
+     *
+     * @param id key in the test table.
+     * @return generated comma-separated line.
+     */
+    public String randomCsvLine(long id) {
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        StringBuilder line = new StringBuilder().append(id);
+
+        for (int vi = 1; vi <= valFieldsCnt; vi++) {
+            line.append(',');
+
+            if (vi % 2 == 1)
+                line.append(rnd.nextLong());
+            else
+                line.append('"').append(rnd.nextLong()).append('"');
+        }
+
+        return line.toString();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
new file mode 100644
index 0000000..f0bdee0
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload.model;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Describes data model.
+ * Matches data model, defined in {@link QueryFactory#createTable()}
+ */
+public class Values10 {
+    /** */
+    final String val1;
+
+    /** */
+    final long val2;
+
+    /** */
+    final String val3;
+
+    /** */
+    final long val4;
+
+    /** */
+    final String val5;
+
+    /** */
+    final long val6;
+
+    /** */
+    final String val7;
+
+    /** */
+    final long val8;
+
+    /** */
+    final String val9;
+
+    /** */
+    final long val10;
+
+    /** Creates new object with randomly initialized fields */
+    public Values10(){
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        val1 = String.valueOf(rnd.nextLong());
+        val2 = rnd.nextLong();
+
+        val3 = String.valueOf(rnd.nextLong());
+        val4 = rnd.nextLong();
+
+        val5 = String.valueOf(rnd.nextLong());
+        val6 = rnd.nextLong();
+
+        val7 = String.valueOf(rnd.nextLong());
+        val8 = rnd.nextLong();
+
+        val9 = String.valueOf(rnd.nextLong());
+        val10 = rnd.nextLong();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/yardstickframework/package-info.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/yardstickframework/package-info.java b/modules/yardstick/src/main/java/org/yardstickframework/package-info.java
new file mode 100644
index 0000000..989a667
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/yardstickframework/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Additional classes for yardstick framework
+ * This package name chosen so, according to default package, yardstick
+ * searches for classes to load
+ */
+
+package org.yardstickframework;
+

http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java b/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
new file mode 100644
index 0000000..e4153ab
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.yardstickframework.probes;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkDriver;
+import org.yardstickframework.BenchmarkExecutionAwareProbe;
+import org.yardstickframework.BenchmarkProbePoint;
+import org.yardstickframework.BenchmarkTotalsOnlyProbe;
+
+import static org.yardstickframework.BenchmarkUtils.println;
+
+/**
+ * Probe that measures total execution time of action for each thread.
+ */
+public class TotalTimeProbe implements BenchmarkExecutionAwareProbe, BenchmarkTotalsOnlyProbe {
+    /** Thread local agents. */
+    private volatile ThreadAgent[] agents;
+
+    /** Benchmark configuration. */
+    private BenchmarkConfiguration cfg;
+
+    /** {@inheritDoc} */
+    @Override public void beforeExecute(int threadIdx) {
+        agents[threadIdx].before();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void afterExecute(int threadIdx) {
+        agents[threadIdx].after();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void start(BenchmarkDriver drv, BenchmarkConfiguration cfg) throws Exception {
+        this.cfg = cfg;
+
+        agents = new ThreadAgent[cfg.threads()];
+
+        for (int thId = 0; thId < agents.length; thId++)
+            agents[thId] = new ThreadAgent();
+
+        println(cfg, "TotalTime probe started");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void stop() throws Exception {
+        println(cfg, "TotalTime probe stopped");
+    }
+
+    /**
+     * This probe producing two-dimension points:
+     * Number of thread that executed {@code test} method and
+     * time it took to finish.
+     *
+     * @return meta information about each point.
+     */
+    @Override public Collection<String> metaInfo() {
+        return Arrays.asList("Thread ID", "Duration, ms");
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<BenchmarkProbePoint> points() {
+        Collection<BenchmarkProbePoint> points = new ArrayList<>();
+
+        for (int thId = 0; thId < agents.length; thId++) {
+
+            double durationMs = agents[thId].getDurationNs() / 1000_000d;
+
+            BenchmarkProbePoint pnt = new BenchmarkProbePoint(thId, new double[] {durationMs});
+
+            points.add(pnt);
+        }
+
+        return points;
+    }
+
+    /**
+     * Useless for {@link BenchmarkTotalsOnlyProbe}.
+     */
+    @Override public void buildPoint(long time) {
+        // No-op
+    }
+
+    /**
+     * Measures execution time. Used in thread local context.
+     */
+    static class ThreadAgent {
+        /** Start time of {@link org.yardstickframework.BenchmarkDriver#test(java.util.Map)} method. */
+        private volatile long startTs;
+
+        /** Finish time. */
+        private volatile long finishTs;
+
+        /**
+         * Start measuring.
+         */
+        public void before(){
+            startTs = System.nanoTime();
+        }
+
+        /**
+         * Stop measuring.
+         */
+        public void after(){
+            finishTs = System.nanoTime();
+        }
+
+        /**
+         * @return measured duration in nanoseconds.
+         */
+        public long getDurationNs(){
+            if (finishTs == 0L && startTs != 0)
+                throw new IllegalStateException("Cannot provide results of TotalTime probe; " +
+                    "after() method haven't been called");
+
+            return (finishTs - startTs);
+        }
+    }
+}


[04/50] [abbrv] ignite git commit: IGNITE-7566 .NET: Build scripts: stop build when one of the steps fails

Posted by sb...@apache.org.
IGNITE-7566 .NET: Build scripts: stop build when one of the steps fails


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/31087002
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/31087002
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/31087002

Branch: refs/heads/ignite-zk
Commit: 31087002b94127c3f1365e90e2a011b3585d085a
Parents: 0936f16
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Wed Mar 7 11:25:07 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Wed Mar 7 11:25:07 2018 +0300

----------------------------------------------------------------------
 modules/platforms/dotnet/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/31087002/modules/platforms/dotnet/build.sh
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/build.sh b/modules/platforms/dotnet/build.sh
index 1a58ff0..66d3852 100755
--- a/modules/platforms/dotnet/build.sh
+++ b/modules/platforms/dotnet/build.sh
@@ -21,7 +21,7 @@
 #
 
 # Fail on error.
-#set -e
+set -e
 
 # Build Java.
 pushd .


[47/50] [abbrv] ignite git commit: IGNITE-7863: Spark dependencies cleaned up. - Fixes #3648.

Posted by sb...@apache.org.
IGNITE-7863: Spark dependencies cleaned up. - Fixes #3648.

Signed-off-by: Nikolay Izhikov <ni...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/98ba7b32
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/98ba7b32
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/98ba7b32

Branch: refs/heads/ignite-zk
Commit: 98ba7b32decfc402deb34c9ed98ec19285fe5648
Parents: 892d8c7
Author: Nikolay Izhikov <ni...@apache.org>
Authored: Sat Mar 17 13:16:11 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Sat Mar 17 13:16:11 2018 +0300

----------------------------------------------------------------------
 modules/spark-2.10/pom.xml | 119 -------------------------------
 modules/spark/pom.xml      | 152 ++--------------------------------------
 2 files changed, 6 insertions(+), 265 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/98ba7b32/modules/spark-2.10/pom.xml
----------------------------------------------------------------------
diff --git a/modules/spark-2.10/pom.xml b/modules/spark-2.10/pom.xml
index f878c9e..1f40f0f 100644
--- a/modules/spark-2.10/pom.xml
+++ b/modules/spark-2.10/pom.xml
@@ -149,102 +149,11 @@
 
         <dependency>
             <groupId>org.apache.spark</groupId>
-            <artifactId>spark-launcher_2.10</artifactId>
-            <version>${spark.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.spark</groupId>
             <artifactId>spark-tags_2.10</artifactId>
             <version>${spark.version}</version>
         </dependency>
 
         <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-lang3</artifactId>
-            <version>3.5</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.netty</groupId>
-            <artifactId>netty-all</artifactId>
-            <version>4.0.29.Final</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>com.esotericsoftware.kryo</groupId>
-            <artifactId>kryo</artifactId>
-            <version>2.20</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>com.twitter</groupId>
-            <artifactId>chill_2.10</artifactId>
-            <version>0.8.1</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>com.codahale.metrics</groupId>
-            <artifactId>metrics-core</artifactId>
-            <version>3.0.2</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.dropwizard.metrics</groupId>
-            <artifactId>metrics-json</artifactId>
-            <version>3.1.2</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.tomcat</groupId>
-            <artifactId>tomcat-servlet-api</artifactId>
-            <version>8.0.23</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.glassfish.jersey.containers</groupId>
-            <artifactId>jersey-container-servlet-core</artifactId>
-            <version>2.25</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>${jackson2.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-scala_2.10</artifactId>
-            <version>${jackson2.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.xbean</groupId>
-            <artifactId>xbean-asm5-shaded</artifactId>
-            <version>4.5</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>net.jpountz.lz4</groupId>
-            <artifactId>lz4</artifactId>
-            <version>1.3.0</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
             <groupId>org.apache.parquet</groupId>
             <artifactId>parquet-hadoop</artifactId>
             <version>1.9.0</version>
@@ -257,34 +166,6 @@
         </dependency>
 
         <dependency>
-            <groupId>org.codehaus.janino</groupId>
-            <artifactId>janino</artifactId>
-            <version>3.0.0</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.codehaus.janino</groupId>
-            <artifactId>commons-compiler</artifactId>
-            <version>3.0.0</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.antlr</groupId>
-            <artifactId>antlr4-runtime</artifactId>
-            <version>4.5.3</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>jul-to-slf4j</artifactId>
-            <version>1.7.16</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
             <version>${guava14.version}</version>

http://git-wip-us.apache.org/repos/asf/ignite/blob/98ba7b32/modules/spark/pom.xml
----------------------------------------------------------------------
diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml
index 3c16996..e338c88 100644
--- a/modules/spark/pom.xml
+++ b/modules/spark/pom.xml
@@ -92,6 +92,12 @@
         </dependency>
 
         <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-tags_2.11</artifactId>
+            <version>${spark.version}</version>
+        </dependency>
+
+        <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${spark.hadoop.version}</version>
@@ -140,152 +146,6 @@
                 </exclusion>
             </exclusions>
         </dependency>
-
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-unsafe_2.11</artifactId>
-            <version>${spark.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-launcher_2.11</artifactId>
-            <version>${spark.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-tags_2.11</artifactId>
-            <version>${spark.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-lang3</artifactId>
-            <version>3.5</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.netty</groupId>
-            <artifactId>netty-all</artifactId>
-            <version>4.0.29.Final</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>com.esotericsoftware.kryo</groupId>
-            <artifactId>kryo</artifactId>
-            <version>2.20</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>com.twitter</groupId>
-            <artifactId>chill_2.11</artifactId>
-            <version>0.8.1</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>com.codahale.metrics</groupId>
-            <artifactId>metrics-core</artifactId>
-            <version>3.0.2</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.dropwizard.metrics</groupId>
-            <artifactId>metrics-json</artifactId>
-            <version>3.1.2</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.tomcat</groupId>
-            <artifactId>tomcat-servlet-api</artifactId>
-            <version>8.0.23</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.glassfish.jersey.containers</groupId>
-            <artifactId>jersey-container-servlet-core</artifactId>
-            <version>2.25</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>${jackson2.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-scala_2.11</artifactId>
-            <version>${jackson2.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.xbean</groupId>
-            <artifactId>xbean-asm5-shaded</artifactId>
-            <version>4.5</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>net.jpountz.lz4</groupId>
-            <artifactId>lz4</artifactId>
-            <version>1.3.0</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.parquet</groupId>
-            <artifactId>parquet-hadoop</artifactId>
-            <version>1.9.0</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-mapreduce-client-core</artifactId>
-            <version>${hadoop.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.codehaus.janino</groupId>
-            <artifactId>janino</artifactId>
-            <version>3.0.0</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.codehaus.janino</groupId>
-            <artifactId>commons-compiler</artifactId>
-            <version>3.0.0</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.antlr</groupId>
-            <artifactId>antlr4-runtime</artifactId>
-            <version>4.5.3</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>jul-to-slf4j</artifactId>
-            <version>1.7.16</version>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 
     <build>


[20/50] [abbrv] ignite git commit: IGNITE-7638 Implemented robin-hood hashing for FullPageIdTable - Fixes #3481.

Posted by sb...@apache.org.
IGNITE-7638 Implemented robin-hood hashing for FullPageIdTable - Fixes #3481.

Signed-off-by: Alexey Goncharuk <al...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1f8ec79d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1f8ec79d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1f8ec79d

Branch: refs/heads/ignite-zk
Commit: 1f8ec79d22305fb5a6e28c9c9fa8efa2a2e391bf
Parents: ac3d545
Author: dpavlov <dp...@gridgain.com>
Authored: Tue Mar 13 16:18:54 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Mar 13 16:18:54 2018 +0300

----------------------------------------------------------------------
 modules/core/pom.xml                            |   7 +
 .../apache/ignite/IgniteSystemProperties.java   |   7 +
 .../GridCacheDatabaseSharedManager.java         |  18 +-
 .../pagemem/CheckpointMetricsTracker.java       |   2 +-
 .../persistence/pagemem/EvictCandidate.java     |  77 ---
 .../persistence/pagemem/FullPageIdTable.java    | 358 +++++-----
 .../persistence/pagemem/LoadedPagesMap.java     | 137 ++++
 .../cache/persistence/pagemem/PageMemoryEx.java |   8 +-
 .../persistence/pagemem/PageMemoryImpl.java     | 192 +++---
 .../persistence/pagemem/ReplaceCandidate.java   |  78 +++
 .../pagemem/RobinHoodBackwardShiftHashMap.java  | 654 +++++++++++++++++++
 .../checkpoint/IgniteMassLoadSandboxTest.java   |   2 +-
 .../pagemem/FullPageIdTableTest.java            | 195 +++++-
 .../RobinHoodBackwardShiftHashMapTest.java      | 430 ++++++++++++
 .../testsuites/IgnitePdsUnitTestSuite.java      |   7 +-
 15 files changed, 1812 insertions(+), 360 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/pom.xml
----------------------------------------------------------------------
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 68c19a7..e56d141 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -124,6 +124,13 @@
         </dependency>
 
         <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-library</artifactId>
+            <version>1.3</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-all</artifactId>
             <version>1.9.5</version>

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index b1a1542..11419ec 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -815,12 +815,19 @@ public final class IgniteSystemProperties {
      * Because other thread may require exactly the same page to be loaded from store, reads are protected by locking.
      */
     public static final String IGNITE_DELAYED_REPLACED_PAGE_WRITE = "IGNITE_DELAYED_REPLACED_PAGE_WRITE";
+
     /**
      * When set to {@code true}, WAL implementation with dedicated worker will be used even in FSYNC mode.
      * Default is {@code false}.
      */
     public static final String IGNITE_WAL_FSYNC_WITH_DEDICATED_WORKER = "IGNITE_WAL_FSYNC_WITH_DEDICATED_WORKER";
 
+    /**
+     * When set to {@code false}, loaded pages implementation is switched to previous version of implementation,
+     * FullPageIdTable. {@code True} value enables 'Robin Hood hashing: backward shift deletion'.
+     * Default is {@code true}.
+     */
+    public static final String IGNITE_LOADED_PAGES_BACKWARD_SHIFT_MAP = "IGNITE_LOADED_PAGES_BACKWARD_SHIFT_MAP";
 
     /**
      * Enforces singleton.

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index d4ba4ef..d6a8a30 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -113,8 +113,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
@@ -141,7 +139,6 @@ import org.apache.ignite.internal.util.lang.GridInClosure3X;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.P3;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
@@ -1251,11 +1248,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
         for (Map.Entry<PageMemoryEx, Collection<Integer>> entry : destroyed.entrySet()) {
             final Collection<Integer> grpIds = entry.getValue();
 
-            clearFuts.add(entry.getKey().clearAsync(new P3<Integer, Long, Integer>() {
-                @Override public boolean apply(Integer grpId, Long pageId, Integer tag) {
-                    return grpIds.contains(grpId);
-                }
-            }, false));
+            clearFuts.add(entry.getKey().clearAsync((grpId, pageIdg) -> grpIds.contains(grpId), false));
         }
 
         for (IgniteInternalFuture<Void> clearFut : clearFuts) {
@@ -1985,12 +1978,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
                             PageMemoryEx pageMem = gId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(gId);
 
-                            pageMem.clearAsync(new P3<Integer, Long, Integer>() {
-                                @Override public boolean apply(Integer cacheId, Long pageId, Integer tag) {
-                                    return cacheId == gId && PageIdUtils.partId(pageId) == pId;
-                                }
-                            }, true).get();
-
+                            pageMem.clearAsync(
+                                (grpId, pageId) -> grpId == gId && PageIdUtils.partId(pageId) == pId,
+                                true).get();
                         }
 
                         break;

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
index e533f41..dbe0f40 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
@@ -70,7 +70,7 @@ public class CheckpointMetricsTracker {
     private long cpEnd;
 
     /**
-     *
+     * Increments counter if copy on write page was written.
      */
     public void onCowPageWritten() {
         COW_PAGES_UPDATER.incrementAndGet(this);

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/EvictCandidate.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/EvictCandidate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/EvictCandidate.java
deleted file mode 100644
index eb172c9..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/EvictCandidate.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence.pagemem;
-
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.util.tostring.GridToStringExclude;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- *
- */
-public class EvictCandidate {
-    /** */
-    private int tag;
-
-    /** */
-    @GridToStringExclude
-    private long relPtr;
-
-    /** */
-    @GridToStringInclude
-    private FullPageId fullId;
-
-    /**
-     * @param tag Tag.
-     * @param relPtr Relative pointer.
-     * @param fullId Full page ID.
-     */
-    public EvictCandidate(int tag, long relPtr, FullPageId fullId) {
-        this.tag = tag;
-        this.relPtr = relPtr;
-        this.fullId = fullId;
-    }
-
-    /**
-     * @return Tag.
-     */
-    public int tag() {
-        return tag;
-    }
-
-    /**
-     * @return Relative pointer.
-     */
-    public long relativePointer() {
-        return relPtr;
-    }
-
-    /**
-     * @return Index.
-     */
-    public FullPageId fullId() {
-        return fullId;
-    }
-
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        return S.toString(EvictCandidate.class, this, "relPtr", U.hexLong(relPtr));
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java
index 19d26ff..7fabfe0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java
@@ -17,13 +17,14 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.pagemem;
 
+import java.util.function.BiConsumer;
 import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.util.GridLongList;
 import org.apache.ignite.internal.util.GridUnsafe;
-import org.apache.ignite.internal.util.lang.GridPredicate3;
+import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiInClosure;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR;
 import static org.apache.ignite.IgniteSystemProperties.getFloat;
@@ -31,7 +32,7 @@ import static org.apache.ignite.IgniteSystemProperties.getFloat;
 /**
  *
  */
-public class FullPageIdTable {
+public class FullPageIdTable implements LoadedPagesMap {
     /** Load factor. */
     private static final float LOAD_FACTOR = getFloat(IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR, 2.5f);
 
@@ -68,6 +69,15 @@ public class FullPageIdTable {
     /** */
     private static final int OUTDATED = -3;
 
+    /** Tag offset from entry base. */
+    private static final int TAG_OFFSET = 4;
+
+    /** Page id offset from entry base. */
+    private static final int PAGE_ID_OFFSET = 8;
+
+    /** Value offset from entry base. */
+    private static final int VALUE_OFFSET = 16;
+
     /** Max size, in elements. */
     protected int capacity;
 
@@ -102,31 +112,21 @@ public class FullPageIdTable {
             clear();
     }
 
-    /**
-     * @return Current number of entries in the map.
-     */
-    public final int size() {
+    /** {@inheritDoc} */
+    @Override public final int size() {
         return GridUnsafe.getInt(valPtr);
     }
 
-    /**
-     * @return Maximum number of entries in the map. This maximum can not be always reached.
-     */
-    public final int capacity() {
+    /** {@inheritDoc} */
+    @Override public final int capacity() {
         return capacity;
     }
 
-    /**
-     * Gets value associated with the given key.
-     *
-     * @param cacheId Cache ID.
-     * @param pageId Page ID.
-     * @return A value associated with the given key.
-     */
-    public long get(int cacheId, long pageId, int tag, long absent, long outdated) {
-        assert assertKey(cacheId, pageId);
+    /** {@inheritDoc} */
+    @Override public long get(int grpId, long pageId, int reqVer, long absent, long outdated) {
+        assert assertKey(grpId, pageId);
 
-        int idx = getKey(cacheId, pageId, tag, false);
+        int idx = getKey(grpId, pageId, reqVer, false);
 
         if (idx == -1)
             return absent;
@@ -137,112 +137,97 @@ public class FullPageIdTable {
         return valueAt(idx);
     }
 
-    /**
-     * Refresh outdated value.
-     *
-     * @param cacheId Cache ID.
-     * @param pageId Page ID.
-     * @param tag Partition tag.
-     * @return A value associated with the given key.
-     */
-    public long refresh(int cacheId, long pageId, int tag) {
-        assert assertKey(cacheId, pageId);
+    /** {@inheritDoc} */
+    @Override public long refresh(int grpId, long pageId, int ver) {
+        assert assertKey(grpId, pageId);
 
-        int idx = getKey(cacheId, pageId, tag, true);
+        int idx = getKey(grpId, pageId, ver, true);
 
-        assert idx >= 0 : "[idx=" + idx + ", tag=" + tag + ", cacheId=" + cacheId +
-            ", pageId=" + U.hexLong(pageId) + ']';
+        if (!(idx >= 0) || !(tagAt(idx) < ver)) {
+            A.ensure(idx >= 0, "[idx=" + idx + ", tag=" + ver + ", cacheId=" + grpId +
+                ", pageId=" + U.hexLong(pageId) + ']');
 
-        assert tagAt(idx) < tag : "[idx=" + idx + ", tag=" + tag + ", cacheId=" + cacheId +
-            ", pageId=" + U.hexLong(pageId) + ", tagAtIdx=" + tagAt(idx) + ']';
+            A.ensure(tagAt(idx) < ver, "[idx=" + idx + ", tag=" + ver + ", cacheId=" + grpId +
+                ", pageId=" + U.hexLong(pageId) + ", tagAtIdx=" + tagAt(idx) + ']');
+        }
 
-        setTagAt(idx, tag);
+        setTagAt(idx, ver);
 
         return valueAt(idx);
     }
 
-    /**
-     * Associates the given key with the given value.
-     *  @param cacheId Cache ID
-     * @param pageId Page ID.
-     * @param value Value to set.
-     */
-    public void put(int cacheId, long pageId, long value, int tag) {
-        assert assertKey(cacheId, pageId);
+    /** {@inheritDoc} */
+    @Override public void put(int grpId, long pageId, long val, int ver) {
+        assert assertKey(grpId, pageId);
 
-        int index = putKey(cacheId, pageId, tag);
+        int idx = putKey(grpId, pageId, ver);
 
-        setValueAt(index, value);
+        setValueAt(idx, val);
     }
 
-    /**
-     * Removes key-value association for the given key.
-     *
-     * @param grpId Cache group ID.
-     * @param pageId Page ID.
-     */
-    public void remove(int grpId, long pageId, int tag) {
+    /** {@inheritDoc} */
+    @Override public boolean remove(int grpId, long pageId) {
         assert assertKey(grpId, pageId);
 
-        int index = removeKey(grpId, pageId, tag);
+        int idx = removeKey(grpId, pageId);
+
+        boolean valRmv = idx >= 0;
+
+        if (valRmv)
+            setValueAt(idx, 0);
 
-        if (index >= 0)
-            setValueAt(index, 0);
+        return valRmv;
     }
 
-    /**
-     * Find nearest value from specified position to the right.
-     *
-     * @param idx Index to start searching from.
-     * @param absent Default value that will be returned if no values present.
-     * @return Closest value to the index and it's partition tag or {@code absent} and -1 if no values found.
-     */
-    public EvictCandidate getNearestAt(final int idx, final long absent) {
-        for (int i = idx; i < capacity + idx; i++) {
-            final int idx2 = i >= capacity ? i - capacity : i;
+    /** {@inheritDoc} */
+    @Override public ReplaceCandidate getNearestAt(final int idxStart) {
+        for (int i = idxStart; i < capacity + idxStart; i++) {
+            final int idx2 = normalizeIndex(i);
 
             if (isValuePresentAt(idx2)) {
                 long base = entryBase(idx2);
 
-                int cacheId = GridUnsafe.getInt(base);
-                int tag = GridUnsafe.getInt(base + 4);
-                long pageId = GridUnsafe.getLong(base + 8);
-                long val = GridUnsafe.getLong(base + 16);
+                int grpId = GridUnsafe.getInt(base);
+                int tag = GridUnsafe.getInt(base + TAG_OFFSET);
+                long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
+                long val = GridUnsafe.getLong(base + VALUE_OFFSET);
 
-                return new EvictCandidate(tag, val, new FullPageId(pageId, cacheId));
+                return new ReplaceCandidate(tag, val, new FullPageId(pageId, grpId));
             }
         }
 
         return null;
     }
 
-    /**
-     * @param idx Index to clear value at.
-     * @param pred Test predicate.
-     * @param absent Value to return if the cell is empty.
-     * @return Value at the given index.
-     */
-    public long clearAt(int idx, GridPredicate3<Integer, Long, Integer> pred, long absent) {
-        long base = entryBase(idx);
+    /** {@inheritDoc} */
+    @Override public GridLongList removeIf(int startIdxToClear, int endIdxToClear, KeyPredicate keyPred) {
+        assert endIdxToClear > startIdxToClear
+            : "Start and end indexes are not consistent: {" + startIdxToClear + ", " + endIdxToClear + "}";
 
-        int grpId = GridUnsafe.getInt(base);
-        int tag = GridUnsafe.getInt(base + 4);
-        long pageId = GridUnsafe.getLong(base + 8);
+        int sz = endIdxToClear - startIdxToClear;
 
-        if ((pageId == REMOVED_PAGE_ID && grpId == REMOVED_CACHE_GRP_ID)
-            || (pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID))
-            return absent;
+        GridLongList list = new GridLongList(sz);
+
+        for (int idx = startIdxToClear; idx < endIdxToClear; idx++) {
+            long base = entryBase(idx);
+
+            int grpId = GridUnsafe.getInt(base);
+            long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
+
+            if (isRemoved(grpId, pageId) || isEmpty(grpId, pageId))
+                continue;
+
+            if (!keyPred.test(grpId, pageId))
+                continue;
 
-        if (pred.apply(grpId, pageId, tag)) {
             long res = valueAt(idx);
 
-            setKeyAt(idx, REMOVED_CACHE_GRP_ID, REMOVED_PAGE_ID);
-            setValueAt(idx, 0);
+            setRemoved(idx);
 
-            return res;
+            list.add(res);
         }
-        else
-            return absent;
+
+        return list;
     }
 
     /**
@@ -253,50 +238,50 @@ public class FullPageIdTable {
     private int putKey(int cacheId, long pageId, int tag) {
         int step = 1;
 
-        int index = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
+        int idx = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
 
-        int foundIndex = -1;
+        int foundIdx = -1;
         int res;
 
         do {
-            res = testKeyAt(index, cacheId, pageId, tag);
+            res = testKeyAt(idx, cacheId, pageId, tag);
 
             if (res == OUTDATED) {
-                foundIndex = index;
+                foundIdx = idx;
 
                 break;
             }
             else if (res == EMPTY) {
-                if (foundIndex == -1)
-                    foundIndex = index;
+                if (foundIdx == -1)
+                    foundIdx = idx;
 
                 break;
             }
             else if (res == REMOVED) {
                 // Must continue search to the first empty slot to ensure there are no duplicate mappings.
-                if (foundIndex == -1)
-                    foundIndex = index;
+                if (foundIdx == -1)
+                    foundIdx = idx;
             }
             else if (res == EQUAL)
-                return index;
+                return idx;
             else
                 assert res == NOT_EQUAL;
 
-            index++;
+            idx++;
 
-            if (index >= capacity)
-                index -= capacity;
+            if (idx >= capacity)
+                idx -= capacity;
         }
         while (++step <= maxSteps);
 
-        if (foundIndex != -1) {
-            setKeyAt(foundIndex, cacheId, pageId);
-            setTagAt(foundIndex, tag);
+        if (foundIdx != -1) {
+            setKeyAt(foundIdx, cacheId, pageId);
+            setTagAt(foundIdx, tag);
 
             if (res != OUTDATED)
                 incrementSize();
 
-            return foundIndex;
+            return foundIdx;
         }
 
         throw new IgniteOutOfMemoryException("No room for a new key");
@@ -305,6 +290,7 @@ public class FullPageIdTable {
     /**
      * @param cacheId Cache ID.
      * @param pageId Page ID.
+     * @param refresh Refresh.
      * @return Key index.
      */
     private int getKey(int cacheId, long pageId, int tag, boolean refresh) {
@@ -335,22 +321,97 @@ public class FullPageIdTable {
     }
 
     /**
+     * @param grpId Cache group ID.
+     * @param pageId Page ID.
+     * @return {@code true} if group & page id indicates cell has state 'Removed'.
+     */
+    private boolean isRemoved(int grpId, long pageId) {
+        return pageId == REMOVED_PAGE_ID && grpId == REMOVED_CACHE_GRP_ID;
+    }
+
+    /**
+     * @param idx cell index, normalized.
+     * @return {@code true} if cell with index idx has state 'Empty'.
+     */
+    private boolean isRemoved(int idx) {
+        long base = entryBase(idx);
+
+        int grpId = GridUnsafe.getInt(base);
+        long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
+
+        return isRemoved(grpId, pageId);
+    }
+
+    /**
+     * Sets cell state to 'Removed' or to 'Empty' if next cell is already 'Empty'.
+     * @param idx cell index, normalized.
+     */
+    private void setRemoved(int idx) {
+        setKeyAt(idx, REMOVED_CACHE_GRP_ID, REMOVED_PAGE_ID);
+
+        setValueAt(idx, 0);
+    }
+
+    /**
+     * @param grpId Cache group ID.
+     * @param pageId Page ID.
+     * @return {@code true} if group & page id indicates cell has state 'Empty'.
+     */
+    private boolean isEmpty(int grpId, long pageId) {
+        return pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID;
+    }
+
+    /**
+     * @param idx cell index, normalized.
+     * @return {@code true} if cell with index idx has state 'Empty'.
+     */
+    private boolean isEmpty(int idx) {
+        long base = entryBase(idx);
+
+        int grpId = GridUnsafe.getInt(base);
+        long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
+
+        return isEmpty(grpId, pageId);
+    }
+
+    /**
+     * Sets cell state to 'Empty'.
+     *
+     * @param idx cell index, normalized.
+     */
+    private void setEmpty(int idx) {
+        setKeyAt(idx, EMPTY_CACHE_GRP_ID, EMPTY_PAGE_ID);
+
+        setValueAt(idx, 0);
+    }
+
+    /**
+     * @param i index probably outsize internal array of cells.
+     * @return corresponding index inside cells array.
+     */
+    private int normalizeIndex(int i) {
+        assert i < 2 * capacity;
+
+        return i < capacity ? i : i - capacity;
+    }
+
+    /**
      * @param cacheId Cache ID.
      * @param pageId Page ID.
      * @return Key index.
      */
-    private int removeKey(int cacheId, long pageId, int tag) {
+    private int removeKey(int cacheId, long pageId) {
         int step = 1;
 
-        int index = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
+        int idx = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
 
-        int foundIndex = -1;
+        int foundIdx = -1;
 
         do {
-            long res = testKeyAt(index, cacheId, pageId, tag);
+            long res = testKeyAt(idx, cacheId, pageId, -1);
 
             if (res == EQUAL || res == OUTDATED) {
-                foundIndex = index;
+                foundIdx = idx;
 
                 break;
             }
@@ -359,20 +420,20 @@ public class FullPageIdTable {
             else
                 assert res == REMOVED || res == NOT_EQUAL;
 
-            index++;
+            idx++;
 
-            if (index >= capacity)
-                index -= capacity;
+            if (idx >= capacity)
+                idx -= capacity;
         }
         while (++step <= maxSteps);
 
-        if (foundIndex != -1) {
-            setKeyAt(foundIndex, REMOVED_CACHE_GRP_ID, REMOVED_PAGE_ID);
+        if (foundIdx != -1) {
+            setRemoved(foundIdx);
 
             decrementSize();
         }
 
-        return foundIndex;
+        return foundIdx;
     }
 
     /**
@@ -383,16 +444,16 @@ public class FullPageIdTable {
         long base = entryBase(index);
 
         int grpId = GridUnsafe.getInt(base);
-        int tag = GridUnsafe.getInt(base + 4);
-        long pageId = GridUnsafe.getLong(base + 8);
+        int tag = GridUnsafe.getInt(base + TAG_OFFSET);
+        long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
 
-        if (pageId == REMOVED_PAGE_ID && grpId == REMOVED_CACHE_GRP_ID)
+        if (isRemoved(grpId, pageId))
             return REMOVED;
         else if (pageId == testPageId && grpId == testCacheId && tag >= testTag)
             return EQUAL;
         else if (pageId == testPageId && grpId == testCacheId && tag < testTag)
             return OUTDATED;
-        else if (pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID)
+        else if (isEmpty(grpId, pageId))
             return EMPTY;
         else
             return NOT_EQUAL;
@@ -406,10 +467,9 @@ public class FullPageIdTable {
         long base = entryBase(idx);
 
         int grpId = GridUnsafe.getInt(base);
-        long pageId = GridUnsafe.getLong(base + 8);
+        long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
 
-        return !((pageId == REMOVED_PAGE_ID && grpId == REMOVED_CACHE_GRP_ID)
-            || (pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID));
+        return !isRemoved(grpId, pageId) && !isEmpty(grpId, pageId);
     }
 
     /**
@@ -425,15 +485,15 @@ public class FullPageIdTable {
     }
 
     /**
-     * @param index Entry index.
+     * @param idx Entry index.
      * @param grpId Cache group ID to write.
      * @param pageId Page ID to write.
      */
-    private void setKeyAt(int index, int grpId, long pageId) {
-        long base = entryBase(index);
+    private void setKeyAt(int idx, int grpId, long pageId) {
+        long base = entryBase(idx);
 
         GridUnsafe.putInt(base, grpId);
-        GridUnsafe.putLong(base + 8, pageId);
+        GridUnsafe.putLong(base + PAGE_ID_OFFSET, pageId);
     }
 
     /**
@@ -478,21 +538,17 @@ public class FullPageIdTable {
         return found ? scans : -scans;
     }
 
-    /**
-     * Scans all the elements in this table.
-     *
-     * @param visitor Visitor.
-     */
-    public void visitAll(IgniteBiInClosure<FullPageId, Long> visitor) {
+    /** {@inheritDoc} */
+    @Override public void forEach(BiConsumer<FullPageId, Long> act) {
         for (int i = 0; i < capacity; i++) {
             if (isValuePresentAt(i)) {
                 long base = entryBase(i);
 
                 int cacheId = GridUnsafe.getInt(base);
-                long pageId = GridUnsafe.getLong(base + 8);
-                long val = GridUnsafe.getLong(base + 16);
+                long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
+                long val = GridUnsafe.getLong(base + VALUE_OFFSET);
 
-                visitor.apply(new FullPageId(pageId, cacheId), val);
+                act.accept(new FullPageId(pageId, cacheId), val);
             }
         }
     }
@@ -502,7 +558,7 @@ public class FullPageIdTable {
      * @return Value.
      */
     private long valueAt(int index) {
-        return GridUnsafe.getLong(entryBase(index) + 16);
+        return GridUnsafe.getLong(entryBase(index) + VALUE_OFFSET);
     }
 
     /**
@@ -510,27 +566,31 @@ public class FullPageIdTable {
      * @param value Value.
      */
     private void setValueAt(int index, long value) {
-        GridUnsafe.putLong(entryBase(index) + 16, value);
+        GridUnsafe.putLong(entryBase(index) + VALUE_OFFSET, value);
     }
 
-    private long entryBase(int index) {
-        return valPtr + 8 + (long)index * BYTES_PER_ENTRY;
+    /**
+     * @param idx Entry index.
+     * @return address of entry.
+     */
+    private long entryBase(int idx) {
+        return valPtr + 8 + (long)idx * BYTES_PER_ENTRY;
     }
 
     /**
-     * @param index Index to get tag for.
+     * @param idx Index to get tag for.
      * @return Tag at the given index.
      */
-    private int tagAt(int index) {
-        return GridUnsafe.getInt(entryBase(index) + 4);
+    private int tagAt(int idx) {
+        return GridUnsafe.getInt(entryBase(idx) + TAG_OFFSET);
     }
 
     /**
-     * @param index Index to set tag for.
+     * @param idx Index to set tag for.
      * @param tag Tag to set at the given index.
      */
-    private void setTagAt(int index, int tag) {
-        GridUnsafe.putInt(entryBase(index) + 4, tag);
+    private void setTagAt(int idx, int tag) {
+        GridUnsafe.putInt(entryBase(idx) + TAG_OFFSET, tag);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/LoadedPagesMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/LoadedPagesMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/LoadedPagesMap.java
new file mode 100644
index 0000000..1b23238
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/LoadedPagesMap.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.util.function.BiConsumer;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.util.GridLongList;
+
+/**
+ * Interface for storing correspondence of page ID in a cache group to long value (address in offheap segment). <br>
+ * Map support versioning of entries. Outdated entry (entry having version lower than requested),
+ * is not provided in case of get, outdated return value is provided instead.<br>
+ *
+ * This mapping is not thread safe. Operations should be protected by outside locking.<br>
+ */
+public interface LoadedPagesMap {
+    /**
+     * Gets value associated with the given key.
+     *
+     * @param grpId Cache Group ID. First part of the key.
+     * @param pageId Page ID. Second part of the key.
+     * @param reqVer Requested entry version, counter associated with value.
+     * @param absent return if provided page is not presented in map.
+     * @param outdated return if provided {@code reqVer} version is greater than value in map (was used for put).
+     * @return A value associated with the given key.
+     */
+    public long get(int grpId, long pageId, int reqVer, long absent, long outdated);
+
+    /**
+     * Associates the given key with the given value.
+     *
+     * @param grpId Cache Group ID. First part of the key.
+     * @param pageId Page ID. Second part of the key.
+     * @param val Value to set.
+     * @param ver Version/counter associated with value, can be used to check if value is outdated.
+     */
+    public void put(int grpId, long pageId, long val, int ver);
+
+    /**
+     * Refresh outdated value. Sets provided version to value associated with cache and page.
+     * Method should be called only for key present and only if version was outdated.
+     * Method may be called in case {@link #get(int, long, int, long, long)} returned {@code outdated} return value.
+     *
+     * @param grpId First part of the key. Cache Group ID.
+     * @param pageId Second part of the key. Page ID.
+     * @param ver Partition tag.
+     * @return A value associated with the given key.
+     * @throws IllegalArgumentException if method is called for absent key or key with fresh version.
+     */
+    public long refresh(int grpId, long pageId, int ver);
+
+    /**
+     * Removes key-value association for the given key.
+     * @param grpId First part of the key. Cache Group ID.
+     * @param pageId Second part of the key. Page ID.
+     * @return {@code True} if value was actually found and removed.
+     */
+    public boolean remove(int grpId, long pageId);
+
+    /**
+     * @return Maximum number of entries in the map. This maximum can not be always reached.
+     */
+    public int capacity();
+
+    /**
+     * @return Current number of entries in the map.
+     */
+    public int size();
+
+    /**
+     * Find nearest presented value from specified position to the right.
+     *
+     * @param idxStart Index to start searching from. Bounded with {@link #capacity()}.
+     * @return Closest value to the index and it's partition tag or  {@code null} value that will
+     * be returned if no values present.
+     */
+    public ReplaceCandidate getNearestAt(int idxStart);
+
+
+    /**
+     * Removes entities matching provided predicate at specified mapping range.
+     *
+     * @param startIdxToClear Index to clear value at, inclusive. Bounded with {@link #capacity()}.
+     * @param endIdxToClear Index to clear value at, inclusive. Bounded with {@link #capacity()}.
+     * @param keyPred Test predicate for (cache group ID, page ID).
+     * @return List with removed values, value is not added to list for empty cell or if key is not matching to
+     * predicate.
+     */
+    public GridLongList removeIf(int startIdxToClear, int endIdxToClear, KeyPredicate keyPred);
+
+
+    /**
+     * Removes entities matching provided predicate.
+     *
+     * @param keyPred Test predicate for (cache group ID, page ID).
+     * @return List with removed values, value is not added to list for empty cell or if key is not matching to
+     * predicate.
+     */
+    default GridLongList removeIf(KeyPredicate keyPred) {
+        return removeIf(0, capacity(), keyPred);
+    }
+
+    /**
+     * Scans all the elements in this table.
+     *
+     * @param act Visitor/action to be applied to each not empty cell.
+     */
+    public void forEach(BiConsumer<FullPageId, Long> act);
+
+    /**
+     * Interface describing a predicate for Key (cache group ID, page ID). Usage of this predicate prevents odd object
+     * creation.
+     */
+    @FunctionalInterface public interface KeyPredicate {
+        /**
+         * Predicate body.
+         *
+         * @param grpId Cache group ID.
+         * @param pageId Page ID.
+         */
+        boolean test(int grpId, long pageId);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
index 9c37508..e889fc5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
@@ -117,7 +117,7 @@ public interface PageMemoryEx extends PageMemory {
      *      the {@link #beginCheckpoint()} method call.
      * @param outBuf Temporary buffer to write changes into.
      * @param tracker Checkpoint metrics tracker.
-     * @return {@code Partition tag} if data was read, {@code null} otherwise (data already saved to storage).
+     * @return {@code Partition generation} if data was read, {@code null} otherwise (data already saved to storage).
      * @throws IgniteException If failed to obtain page data.
      */
     @Nullable public Integer getForCheckpoint(FullPageId pageId, ByteBuffer outBuf, CheckpointMetricsTracker tracker);
@@ -127,7 +127,7 @@ public interface PageMemoryEx extends PageMemory {
      *
      * @param grpId Group ID.
      * @param partId Partition ID.
-     * @return New partition tag (growing 1-based partition file version).
+     * @return New partition generation (growing 1-based partition file version).
      */
     public int invalidate(int grpId, int partId);
 
@@ -141,9 +141,9 @@ public interface PageMemoryEx extends PageMemory {
     /**
      * Asynchronously clears pages satisfying the given predicate.
      *
-     * @param pred Predicate for cache group id, pageId and partition tag.
+     * @param pred Predicate for cache group id, pageId.
      * @param cleanDirty Flag indicating that dirty pages collection should be cleaned.
      * @return Future that will be completed when all pages are cleared.
      */
-    public IgniteInternalFuture<Void> clearAsync(GridPredicate3<Integer, Long, Integer> pred, boolean cleanDirty);
+    public IgniteInternalFuture<Void> clearAsync(LoadedPagesMap.KeyPredicate pred, boolean cleanDirty);
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 27c9b38..fa10a1f 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -23,7 +23,6 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -38,6 +37,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -62,6 +62,7 @@ import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockSta
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointWriteProgressSupplier;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionCountersIO;
@@ -76,9 +77,7 @@ import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.OffheapReadWriteLock;
 import org.apache.ignite.internal.util.future.CountDownFuture;
 import org.apache.ignite.internal.util.lang.GridInClosure3X;
-import org.apache.ignite.internal.util.lang.GridPredicate3;
 import org.apache.ignite.internal.util.offheap.GridOffHeapOutOfMemoryException;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.jetbrains.annotations.Nullable;
@@ -157,7 +156,7 @@ public class PageMemoryImpl implements PageMemoryEx {
     /** Page lock offset. */
     public static final int PAGE_LOCK_OFFSET = 32;
 
-    /** Page lock offset. */
+    /** Page temp copy buffer relative pointer offset. */
     public static final int PAGE_TMP_BUF_OFFSET = 40;
 
     /**
@@ -189,6 +188,10 @@ public class PageMemoryImpl implements PageMemoryEx {
     /** Number of used pages in checkpoint buffer. */
     private final AtomicInteger cpBufPagesCntr = new AtomicInteger(0);
 
+    /** Use new implementation of loaded pages table:  'Robin Hood hashing: backward shift deletion'. */
+    private final boolean useBackwardShiftMap
+        = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_LOADED_PAGES_BACKWARD_SHIFT_MAP, true);
+
     /** */
     private ExecutorService asyncRunner = new ThreadPoolExecutor(
         0,
@@ -483,7 +486,7 @@ public class PageMemoryImpl implements PageMemoryEx {
             long relPtr = seg.loadedPages.get(
                 cacheId,
                 PageIdUtils.effectivePageId(pageId),
-                seg.partTag(cacheId, partId),
+                seg.partGeneration(cacheId, partId),
                 INVALID_REL_PTR,
                 OUTDATED_REL_PTR
             );
@@ -536,7 +539,7 @@ public class PageMemoryImpl implements PageMemoryEx {
                 }
             }
 
-            seg.loadedPages.put(cacheId, PageIdUtils.effectivePageId(pageId), relPtr, seg.partTag(cacheId, partId));
+            seg.loadedPages.put(cacheId, PageIdUtils.effectivePageId(pageId), relPtr, seg.partGeneration(cacheId, partId));
         }
         catch (IgniteOutOfMemoryException oom) {
             DataRegionConfiguration dataRegionCfg = getDataRegionConfiguration();
@@ -628,7 +631,7 @@ public class PageMemoryImpl implements PageMemoryEx {
             long relPtr = seg.loadedPages.get(
                 cacheId,
                 PageIdUtils.effectivePageId(pageId),
-                seg.partTag(cacheId, partId),
+                seg.partGeneration(cacheId, partId),
                 INVALID_REL_PTR,
                 INVALID_REL_PTR
             );
@@ -659,7 +662,7 @@ public class PageMemoryImpl implements PageMemoryEx {
             long relPtr = seg.loadedPages.get(
                 cacheId,
                 PageIdUtils.effectivePageId(pageId),
-                seg.partTag(cacheId, partId),
+                seg.partGeneration(cacheId, partId),
                 INVALID_REL_PTR,
                 OUTDATED_REL_PTR
             );
@@ -688,7 +691,7 @@ public class PageMemoryImpl implements PageMemoryEx {
                     cacheId,
                     PageIdUtils.effectivePageId(pageId),
                     relPtr,
-                    seg.partTag(cacheId, partId)
+                    seg.partGeneration(cacheId, partId)
                 );
 
                 long pageAddr = absPtr + PAGE_OVERHEAD;
@@ -788,7 +791,7 @@ public class PageMemoryImpl implements PageMemoryEx {
     private long refreshOutdatedPage(Segment seg, int cacheId, long pageId, boolean rmv) {
         assert seg.writeLock().isHeldByCurrentThread();
 
-        int tag = seg.partTag(cacheId, PageIdUtils.partId(pageId));
+        int tag = seg.partGeneration(cacheId, PageIdUtils.partId(pageId));
 
         long relPtr = seg.loadedPages.refresh(cacheId, PageIdUtils.effectivePageId(pageId), tag);
 
@@ -812,7 +815,7 @@ public class PageMemoryImpl implements PageMemoryEx {
         }
 
         if (rmv)
-            seg.loadedPages.remove(cacheId, PageIdUtils.effectivePageId(pageId), tag);
+            seg.loadedPages.remove(cacheId, PageIdUtils.effectivePageId(pageId));
 
         if (seg.segCheckpointPages != null)
             seg.segCheckpointPages.remove(new FullPageId(pageId, cacheId));
@@ -1026,7 +1029,7 @@ public class PageMemoryImpl implements PageMemoryEx {
 
         int tag;
 
-        boolean tmpBuffer = false;
+        boolean pageSingleAcquire = false;
 
         seg.readLock().lock();
 
@@ -1034,7 +1037,7 @@ public class PageMemoryImpl implements PageMemoryEx {
             if (!isInCheckpoint(fullId))
                 return null;
 
-            tag = seg.partTag(fullId.groupId(), PageIdUtils.partId(fullId.pageId()));
+            tag = seg.partGeneration(fullId.groupId(), PageIdUtils.partId(fullId.pageId()));
 
             relPtr = seg.loadedPages.get(
                 fullId.groupId(),
@@ -1055,7 +1058,7 @@ public class PageMemoryImpl implements PageMemoryEx {
                 if (PageHeader.tempBufferPointer(absPtr) == INVALID_REL_PTR)
                     PageHeader.acquirePage(absPtr);
                 else
-                    tmpBuffer = true;
+                    pageSingleAcquire = true;
             }
         }
         finally {
@@ -1070,7 +1073,7 @@ public class PageMemoryImpl implements PageMemoryEx {
                 relPtr = seg.loadedPages.get(
                     fullId.groupId(),
                     PageIdUtils.effectivePageId(fullId.pageId()),
-                    seg.partTag(
+                    seg.partGeneration(
                         fullId.groupId(),
                         PageIdUtils.partId(fullId.pageId())
                     ),
@@ -1099,19 +1102,22 @@ public class PageMemoryImpl implements PageMemoryEx {
             }
         }
         else
-            return copyPageForCheckpoint(absPtr, fullId, outBuf, tmpBuffer, tracker) ? tag : null;
+            return copyPageForCheckpoint(absPtr, fullId, outBuf, pageSingleAcquire, tracker) ? tag : null;
     }
 
     /**
      * @param absPtr Absolute ptr.
      * @param fullId Full id.
-     * @param tmpBuf Tmp buffer.
+     * @param outBuf Output buffer to write page content into.
+     * @param pageSingleAcquire Page is acquired only once. We don't pin the page second time (until page will not be
+     * copied) in case checkpoint temporary buffer is used.
+     * @param tracker Checkpoint statistics tracker.
      */
     private boolean copyPageForCheckpoint(
         long absPtr,
         FullPageId fullId,
-        ByteBuffer tmpBuf,
-        boolean tmpBuffer,
+        ByteBuffer outBuf,
+        boolean pageSingleAcquire,
         CheckpointMetricsTracker tracker
     ) {
         assert absPtr != 0;
@@ -1131,7 +1137,7 @@ public class PageMemoryImpl implements PageMemoryEx {
 
                 long tmpAbsPtr = checkpointPool.absolute(tmpRelPtr);
 
-                copyInBuffer(tmpAbsPtr, tmpBuf);
+                copyInBuffer(tmpAbsPtr, outBuf);
 
                 GridUnsafe.setMemory(tmpAbsPtr + PAGE_OVERHEAD, pageSize(), (byte)0);
 
@@ -1145,12 +1151,12 @@ public class PageMemoryImpl implements PageMemoryEx {
 
                 // Need release again because we pin page when resolve abs pointer,
                 // and page did not have tmp buffer page.
-                if (!tmpBuffer)
+                if (!pageSingleAcquire)
                     PageHeader.releasePage(absPtr);
 
             }
             else {
-                copyInBuffer(absPtr, tmpBuf);
+                copyInBuffer(absPtr, outBuf);
 
                 PageHeader.dirty(absPtr, false);
 
@@ -1158,8 +1164,8 @@ public class PageMemoryImpl implements PageMemoryEx {
                 PageHeader.releasePage(absPtr);
             }
 
-            assert PageIO.getType(tmpBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
-            assert PageIO.getVersion(tmpBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(fullId.pageId());
+            assert PageIO.getType(outBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
+            assert PageIO.getVersion(outBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(fullId.pageId());
 
             return true;
         }
@@ -1199,7 +1205,7 @@ public class PageMemoryImpl implements PageMemoryEx {
             seg.writeLock().lock();
 
             try {
-                int newTag = seg.incrementPartTag(grpId, partId);
+                int newTag = seg.incrementPartGeneration(grpId, partId);
 
                 if (tag == 0)
                     tag = newTag;
@@ -1220,7 +1226,7 @@ public class PageMemoryImpl implements PageMemoryEx {
             seg.writeLock().lock();
 
             try {
-                seg.resetPartTags(grpId);
+                seg.resetGroupPartitionsGeneration(grpId);
             }
             finally {
                 seg.writeLock().unlock();
@@ -1230,7 +1236,7 @@ public class PageMemoryImpl implements PageMemoryEx {
 
     /** {@inheritDoc} */
     @Override public IgniteInternalFuture<Void> clearAsync(
-        GridPredicate3<Integer, Long, Integer> pred,
+        LoadedPagesMap.KeyPredicate pred,
         boolean cleanDirty
     ) {
         CountDownFuture completeFut = new CountDownFuture(segments.length);
@@ -1811,16 +1817,22 @@ public class PageMemoryImpl implements PageMemoryEx {
         /** */
         private static final double FULL_SCAN_THRESHOLD = 0.4;
 
+        /** Pointer to acquired pages integer counter. */
+        private static final int ACQUIRED_PAGES_SIZEOF = 4;
+
+        /** Padding to read from word beginning. */
+        private static final int ACQUIRED_PAGES_PADDING = 4;
+
         /** Page ID to relative pointer map. */
-        private FullPageIdTable loadedPages;
+        private LoadedPagesMap loadedPages;
 
-        /** */
+        /** Pointer to acquired pages integer counter. */
         private long acquiredPagesPtr;
 
         /** */
         private PagePool pool;
 
-        /** */
+        /** Bytes required to store {@link #loadedPages}. */
         private long memPerTbl;
 
         /** Pages marked as dirty since the last checkpoint. */
@@ -1832,8 +1844,11 @@ public class PageMemoryImpl implements PageMemoryEx {
         /** */
         private final int maxDirtyPages;
 
-        /** Maps partition (cacheId, partId) to its tag. Tag is 1-based incrementing partition file counter */
-        private final Map<T2<Integer, Integer>, Integer> partTagMap = new HashMap<>();
+        /** Initial partition generation. */
+        private static final int INIT_PART_GENERATION = 1;
+
+        /** Maps partition (cacheId, partId) to its generation. Generation is 1-based incrementing partition counter. */
+        private final Map<GroupPartitionId, Integer> partGenerationMap = new HashMap<>();
 
         /** */
         private boolean closed;
@@ -1847,15 +1862,23 @@ public class PageMemoryImpl implements PageMemoryEx {
 
             int pages = (int)(totalMemory / sysPageSize);
 
-            memPerTbl = requiredSegmentTableMemory(pages);
-
             acquiredPagesPtr = region.address();
 
             GridUnsafe.putIntVolatile(null, acquiredPagesPtr, 0);
 
-            loadedPages = new FullPageIdTable(region.address() + 8, memPerTbl, true);
+            int ldPagesMapOffInRegion = ACQUIRED_PAGES_SIZEOF + ACQUIRED_PAGES_PADDING;
+
+            long ldPagesAddr = region.address() + ldPagesMapOffInRegion;
+
+            memPerTbl = useBackwardShiftMap
+                ? RobinHoodBackwardShiftHashMap.requiredMemory(pages)
+                : requiredSegmentTableMemory(pages);
 
-            DirectMemoryRegion poolRegion = region.slice(memPerTbl + 8);
+            loadedPages = useBackwardShiftMap
+                ? new RobinHoodBackwardShiftHashMap(ldPagesAddr, memPerTbl)
+                : new FullPageIdTable(ldPagesAddr, memPerTbl, true);
+
+            DirectMemoryRegion poolRegion = region.slice(memPerTbl + ldPagesMapOffInRegion);
 
             pool = new PagePool(idx, poolRegion, null);
 
@@ -1980,7 +2003,7 @@ public class PageMemoryImpl implements PageMemoryEx {
                     saveDirtyPage.writePage(
                         fullPageId,
                         wrapPointer(absPtr + PAGE_OVERHEAD, pageSize()),
-                        partTag(
+                        partGeneration(
                             fullPageId.groupId(),
                             PageIdUtils.partId(fullPageId.pageId())
                         )
@@ -2102,13 +2125,13 @@ public class PageMemoryImpl implements PageMemoryEx {
                     // We need to lookup for pages only in current segment for thread safety,
                     // so peeking random memory will lead to checking for found page segment.
                     // It's much faster to check available pages for segment right away.
-                    EvictCandidate nearest = loadedPages.getNearestAt(rnd.nextInt(cap), INVALID_REL_PTR);
+                    ReplaceCandidate nearest = loadedPages.getNearestAt(rnd.nextInt(cap));
 
                     assert nearest != null && nearest.relativePointer() != INVALID_REL_PTR;
 
                     long rndAddr = nearest.relativePointer();
 
-                    int partTag = nearest.tag();
+                    int partGen = nearest.generation();
 
                     final long absPageAddr = absolute(rndAddr);
 
@@ -2118,7 +2141,7 @@ public class PageMemoryImpl implements PageMemoryEx {
                     assert fullId.equals(nearest.fullId()) : "Invalid page mapping [tableId=" + nearest.fullId() +
                         ", actual=" + fullId + ", nearest=" + nearest;
 
-                    boolean outdated = partTag < partTag(fullId.groupId(), PageIdUtils.partId(fullId.pageId()));
+                    boolean outdated = partGen < partGeneration(fullId.groupId(), PageIdUtils.partId(fullId.pageId()));
 
                     if (outdated)
                         return refreshOutdatedPage(this, fullId.groupId(), fullId.pageId(), true);
@@ -2184,11 +2207,7 @@ public class PageMemoryImpl implements PageMemoryEx {
 
                 loadedPages.remove(
                     fullPageId.groupId(),
-                    PageIdUtils.effectivePageId(fullPageId.pageId()),
-                    partTag(
-                        fullPageId.groupId(),
-                        PageIdUtils.partId(fullPageId.pageId())
-                    )
+                    PageIdUtils.effectivePageId(fullPageId.pageId())
                 );
 
                 return relRmvAddr;
@@ -2231,19 +2250,19 @@ public class PageMemoryImpl implements PageMemoryEx {
             int failToPrepare = 0;
 
             for (int i = 0; i < cap; i++) {
-                final EvictCandidate nearest = loadedPages.getNearestAt(i, INVALID_REL_PTR);
+                final ReplaceCandidate nearest = loadedPages.getNearestAt(i);
 
                 assert nearest != null && nearest.relativePointer() != INVALID_REL_PTR;
 
                 final long addr = nearest.relativePointer();
 
-                int partTag = nearest.tag();
+                int partGen = nearest.generation();
 
                 final long absPageAddr = absolute(addr);
 
                 FullPageId fullId = PageHeader.fullPageId(absPageAddr);
 
-                if (partTag < partTag(fullId.groupId(), PageIdUtils.partId(fullId.pageId())))
+                if (partGen < partGeneration(fullId.groupId(), PageIdUtils.partId(fullId.pageId())))
                     return refreshOutdatedPage(this, fullId.groupId(), fullId.pageId(), true);
 
                 boolean pinned = PageHeader.isAcquired(absPageAddr);
@@ -2261,11 +2280,7 @@ public class PageMemoryImpl implements PageMemoryEx {
                 if (preparePageRemoval(fullPageId, absEvictAddr, saveDirtyPage)) {
                     loadedPages.remove(
                         fullPageId.groupId(),
-                        PageIdUtils.effectivePageId(fullPageId.pageId()),
-                        partTag(
-                            fullPageId.groupId(),
-                            PageIdUtils.partId(fullPageId.pageId())
-                        )
+                        PageIdUtils.effectivePageId(fullPageId.pageId())
                     );
 
                     return addr;
@@ -2309,64 +2324,55 @@ public class PageMemoryImpl implements PageMemoryEx {
         /**
          * @param grpId Cache group ID.
          * @param partId Partition ID.
-         * @return Partition tag. Growing 1 based partition file version
+         * @return Partition generation. Growing, 1-based partition version. Changed
          */
-        private int partTag(int grpId, int partId) {
+        private int partGeneration(int grpId, int partId) {
             assert getReadHoldCount() > 0 || getWriteHoldCount() > 0;
 
-            Integer tag = partTagMap.get(new T2<>(grpId, partId));
-
-            if (tag == null)
-                return 1;
+            Integer tag = partGenerationMap.get(new GroupPartitionId(grpId, partId));
 
-            return tag;
+            return tag == null ? INIT_PART_GENERATION : tag;
         }
 
         /**
+         * Increments partition generation due to partition invalidation (e.g. partition was rebalanced to other node
+         * and evicted).
+         *
          * @param grpId Cache group ID.
          * @param partId Partition ID.
-         * @return New partition tag.
+         * @return New partition generation.
          */
-        private int incrementPartTag(int grpId, int partId) {
+        private int incrementPartGeneration(int grpId, int partId) {
             assert getWriteHoldCount() > 0;
 
-            T2<Integer, Integer> t = new T2<>(grpId, partId);
+            GroupPartitionId grpPart = new GroupPartitionId(grpId, partId);
 
-            Integer tag = partTagMap.get(t);
+            Integer gen = partGenerationMap.get(grpPart);
 
-            if (tag == null) {
-                partTagMap.put(t, 2);
+            if (gen == null)
+                gen = INIT_PART_GENERATION;
 
-                return 2;
-            }
-            else if (tag == Integer.MAX_VALUE) {
+            if (gen == Integer.MAX_VALUE) {
                 U.warn(log, "Partition tag overflow [grpId=" + grpId + ", partId=" + partId + "]");
 
-                partTagMap.put(t, 0);
+                partGenerationMap.put(grpPart, 0);
 
                 return 0;
             }
             else {
-                partTagMap.put(t, tag + 1);
+                partGenerationMap.put(grpPart, gen + 1);
 
-                return tag + 1;
+                return gen + 1;
             }
         }
 
         /**
          * @param grpId Cache group id.
          */
-        private void resetPartTags(int grpId) {
+        private void resetGroupPartitionsGeneration(int grpId) {
             assert getWriteHoldCount() > 0;
 
-            Iterator<T2<Integer, Integer>> iter = partTagMap.keySet().iterator();
-
-            while (iter.hasNext()) {
-                T2<Integer, Integer> t = iter.next();
-
-                if (t.get1() == grpId)
-                    iter.remove();
-            }
+            partGenerationMap.keySet().removeIf(grpPart -> grpPart.getGroupId() == grpId);
         }
     }
 
@@ -2552,14 +2558,19 @@ public class PageMemoryImpl implements PageMemoryEx {
         }
 
         /**
+         * Sets pointer to checkpoint buffer.
+         *
          * @param absPtr Page absolute pointer.
-         * @param tmpRelPtr Temp buffer relative pointer.
+         * @param tmpRelPtr Temp buffer relative pointer or {@link #INVALID_REL_PTR} if page is not copied to checkpoint
+         * buffer.
          */
         private static void tempBufferPointer(long absPtr, long tmpRelPtr) {
             GridUnsafe.putLong(absPtr + PAGE_TMP_BUF_OFFSET, tmpRelPtr);
         }
 
         /**
+         * Gets pointer to checkpoint buffer or {@link #INVALID_REL_PTR} if page is not copied to checkpoint buffer.
+         *
          * @param absPtr Page absolute pointer.
          * @return Temp buffer relative pointer.
          */
@@ -2637,8 +2648,8 @@ public class PageMemoryImpl implements PageMemoryEx {
         /** */
         private Segment seg;
 
-        /** */
-        private GridPredicate3<Integer, Long, Integer> clearPred;
+        /** Clear element filter for (cache group ID, page ID). */
+        LoadedPagesMap.KeyPredicate clearPred;
 
         /** */
         private CountDownFuture doneFut;
@@ -2651,12 +2662,12 @@ public class PageMemoryImpl implements PageMemoryEx {
 
         /**
          * @param seg Segment.
-         * @param clearPred Clear predicate.
+         * @param clearPred Clear predicate for (cache group ID, page ID).
          * @param doneFut Completion future.
          */
         private ClearSegmentRunnable(
             Segment seg,
-            GridPredicate3<Integer, Long, Integer> clearPred,
+            LoadedPagesMap.KeyPredicate clearPred,
             boolean rmvDirty,
             CountDownFuture doneFut,
             int pageSize
@@ -2682,14 +2693,11 @@ public class PageMemoryImpl implements PageMemoryEx {
                     seg.writeLock().lock();
 
                     try {
-                        while (base < boundary) {
-                            long ptr = seg.loadedPages.clearAt(base, clearPred, INVALID_REL_PTR);
+                        GridLongList list = seg.loadedPages.removeIf(base, boundary, clearPred);
 
-                            if (ptr != INVALID_REL_PTR)
-                                ptrs.add(ptr);
+                        ptrs.addAll(list);
 
-                            base++;
-                        }
+                        base = boundary;
                     }
                     finally {
                         seg.writeLock().unlock();

http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/ReplaceCandidate.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/ReplaceCandidate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/ReplaceCandidate.java
new file mode 100644
index 0000000..d760f07
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/ReplaceCandidate.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Replacement removal candidate. Class represents some page from loaded pages table. Usually candidate is found during
+ * random {@link LoadedPagesMap} touch.
+ */
+public class ReplaceCandidate {
+    /** Partition generation saved in map, too old value means page may be safely cleared. */
+    private int gen;
+
+    /** */
+    @GridToStringExclude
+    private long relPtr;
+
+    /** */
+    @GridToStringInclude
+    private FullPageId fullId;
+
+    /**
+     * @param gen Partition generation.
+     * @param relPtr Relative pointer to page.
+     * @param fullId Full page ID.
+     */
+    public ReplaceCandidate(int gen, long relPtr, FullPageId fullId) {
+        this.gen = gen;
+        this.relPtr = relPtr;
+        this.fullId = fullId;
+    }
+
+    /**
+     * @return Partition generation saved in map, too old value means page may be safely cleared.
+     */
+    public int generation() {
+        return gen;
+    }
+
+    /**
+     * @return Relative pointer to page.
+     */
+    public long relativePointer() {
+        return relPtr;
+    }
+
+    /**
+     * @return Index.
+     */
+    public FullPageId fullId() {
+        return fullId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ReplaceCandidate.class, this, "relPtr", U.hexLong(relPtr));
+    }
+}


[13/50] [abbrv] ignite git commit: IGNITE-7876: Adopt SVM Linear Binary Classification Model and Trainer to the new Partitioned Dataset

Posted by sb...@apache.org.
IGNITE-7876: Adopt SVM Linear Binary Classification Model and Trainer to the new Partitioned Dataset

This closes #3607


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/318ffe50
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/318ffe50
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/318ffe50

Branch: refs/heads/ignite-zk
Commit: 318ffe50fc0edb1d31f05044be8367bd578b6a88
Parents: 29d56ae
Author: zaleslaw <za...@gmail.com>
Authored: Mon Mar 12 15:28:11 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Mon Mar 12 15:28:11 2018 +0300

----------------------------------------------------------------------
 .../ml/svm/SVMBinaryClassificationExample.java  |  131 --
 .../ml/svm/SVMMultiClassificationExample.java   |  130 --
 .../src/main/resources/datasets/titanic.txt     | 1309 ------------------
 .../apache/ignite/ml/structures/Dataset.java    |   10 +
 .../ignite/ml/structures/LabeledDataset.java    |   16 +-
 .../svm/SVMLinearBinaryClassificationModel.java |    4 +-
 .../SVMLinearBinaryClassificationTrainer.java   |   87 +-
 .../SVMLinearMultiClassClassificationModel.java |   89 --
 ...VMLinearMultiClassClassificationTrainer.java |  160 ---
 .../ignite/ml/svm/SVMPartitionContext.java      |   28 +
 .../ml/svm/SVMPartitionDataBuilderOnHeap.java   |   86 ++
 .../org/apache/ignite/ml/LocalModelsTest.java   |   28 -
 .../org/apache/ignite/ml/svm/BaseSVMTest.java   |   58 -
 .../ignite/ml/svm/SVMBinaryTrainerTest.java     |   74 +
 .../org/apache/ignite/ml/svm/SVMModelTest.java  |   15 -
 .../org/apache/ignite/ml/svm/SVMTestSuite.java  |   13 +-
 ...inearSVMBinaryClassificationTrainerTest.java |   35 -
 ...inearSVMBinaryClassificationTrainerTest.java |  141 --
 ...inearSVMBinaryClassificationTrainerTest.java |   38 -
 ...rSVMMultiClassClassificationTrainerTest.java |   35 -
 ...rSVMMultiClassClassificationTrainerTest.java |   76 -
 ...rSVMMultiClassClassificationTrainerTest.java |   38 -
 22 files changed, 272 insertions(+), 2329 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMBinaryClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMBinaryClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMBinaryClassificationExample.java
deleted file mode 100644
index e256276..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMBinaryClassificationExample.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.svm;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Path;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.examples.ExampleNodeStartup;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.structures.LabeledDataset;
-import org.apache.ignite.ml.structures.LabeledDatasetTestTrainPair;
-import org.apache.ignite.ml.structures.preprocessing.LabeledDatasetLoader;
-import org.apache.ignite.ml.structures.preprocessing.LabellingMachine;
-import org.apache.ignite.ml.structures.preprocessing.Normalizer;
-import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
-import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationTrainer;
-import org.apache.ignite.thread.IgniteThread;
-
-/**
- * <p>
- * Example of using {@link org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel} with Titanic dataset.</p>
- * <p>
- * Note that in this example we cannot guarantee order in which nodes return results of intermediate
- * computations and therefore algorithm can return different results.</p>
- * <p>
- * Remote nodes should always be started with special configuration file which
- * enables P2P class loading: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will start node
- * with {@code examples/config/example-ignite.xml} configuration.</p>
- */
-public class SVMBinaryClassificationExample {
-    /** Separator. */
-    private static final String SEPARATOR = ",";
-
-    /** Path to the Iris dataset. */
-    private static final String TITANIC_DATASET = "examples/src/main/resources/datasets/titanic.txt";
-
-    /**
-     * Executes example.
-     *
-     * @param args Command line arguments, none required.
-     */
-    public static void main(String[] args) throws InterruptedException {
-        System.out.println(">>> SVM Binary classification example started.");
-        // Start ignite grid.
-        try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
-            System.out.println(">>> Ignite grid started.");
-
-            IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-                SVMBinaryClassificationExample.class.getSimpleName(), () -> {
-
-                try {
-                    // Prepare path to read
-                    File file = IgniteUtils.resolveIgnitePath(TITANIC_DATASET);
-                    if (file == null)
-                        throw new RuntimeException("Can't find file: " + TITANIC_DATASET);
-
-                    Path path = file.toPath();
-
-                    // Read dataset from file
-                    LabeledDataset dataset = LabeledDatasetLoader.loadFromTxtFile(path, SEPARATOR, true, false);
-
-                    // Normalize dataset
-                    Normalizer.normalizeWithMiniMax(dataset);
-
-                    // Random splitting of the given data as 70% train and 30% test datasets
-                    LabeledDatasetTestTrainPair split = new LabeledDatasetTestTrainPair(dataset, 0.3);
-
-                    System.out.println("\n>>> Amount of observations in train dataset " + split.train().rowSize());
-                    System.out.println("\n>>> Amount of observations in test dataset " + split.test().rowSize());
-
-                    LabeledDataset test = split.test();
-                    LabeledDataset train = split.train();
-
-                    System.out.println("\n>>> Create new linear binary SVM trainer object.");
-                    Trainer<SVMLinearBinaryClassificationModel, LabeledDataset> trainer = new SVMLinearBinaryClassificationTrainer();
-
-                    System.out.println("\n>>> Perform the training to get the model.");
-                    SVMLinearBinaryClassificationModel mdl = trainer.train(train);
-
-                    System.out.println("\n>>> SVM classification model: " + mdl);
-
-                    // Clone labels
-                    final double[] labels = test.labels();
-
-                    // Save predicted classes to test dataset
-                    LabellingMachine.assignLabels(test, mdl);
-
-                    // Calculate amount of errors on test dataset
-                    int amountOfErrors = 0;
-                    for (int i = 0; i < test.rowSize(); i++) {
-                        if (test.label(i) != labels[i])
-                            amountOfErrors++;
-                    }
-
-                    System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
-                    System.out.println("\n>>> Prediction percentage " + (1 - amountOfErrors / (double) test.rowSize()));
-
-                } catch (IOException e) {
-                    e.printStackTrace();
-                    System.out.println("\n>>> Unexpected exception, check resources: " + e);
-                } finally {
-                    System.out.println("\n>>> SVM binary classification example completed.");
-                }
-
-            });
-
-            igniteThread.start();
-            igniteThread.join();
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMMultiClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMMultiClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMMultiClassificationExample.java
deleted file mode 100644
index 7aee5d2..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMMultiClassificationExample.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.svm;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Path;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.examples.ExampleNodeStartup;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.structures.LabeledDataset;
-import org.apache.ignite.ml.structures.LabeledDatasetTestTrainPair;
-import org.apache.ignite.ml.structures.preprocessing.LabeledDatasetLoader;
-import org.apache.ignite.ml.structures.preprocessing.LabellingMachine;
-import org.apache.ignite.ml.structures.preprocessing.Normalizer;
-import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationModel;
-import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationTrainer;
-import org.apache.ignite.thread.IgniteThread;
-
-/**
- * <p>
- * Example of using {@link org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationModel} with Iris dataset.</p>
- * <p>
- * Note that in this example we cannot guarantee order in which nodes return results of intermediate
- * computations and therefore algorithm can return different results.</p>
- * <p>
- * Remote nodes should always be started with special configuration file which
- * enables P2P class loading: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will start node
- * with {@code examples/config/example-ignite.xml} configuration.</p>
- */
-public class SVMMultiClassificationExample {
-    /** Separator. */
-    private static final String SEPARATOR = "\t";
-
-    /** Path to the Iris dataset. */
-    private static final String IRIS_DATASET = "examples/src/main/resources/datasets/iris.txt";
-
-    /**
-     * Executes example.
-     *
-     * @param args Command line arguments, none required.
-     */
-    public static void main(String[] args) throws InterruptedException {
-        System.out.println(">>> SVM Multi-Class classification example started.");
-        // Start ignite grid.
-        try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
-            System.out.println(">>> Ignite grid started.");
-
-            IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-                SVMMultiClassificationExample.class.getSimpleName(), () -> {
-
-                try {
-                    // Prepare path to read
-                    File file = IgniteUtils.resolveIgnitePath(IRIS_DATASET);
-                    if (file == null)
-                        throw new RuntimeException("Can't find file: " + IRIS_DATASET);
-
-                    Path path = file.toPath();
-
-                    // Read dataset from file
-                    LabeledDataset dataset = LabeledDatasetLoader.loadFromTxtFile(path, SEPARATOR, true, false);
-
-                    // Normalize dataset
-                    Normalizer.normalizeWithMiniMax(dataset);
-
-                    // Random splitting of the given data as 70% train and 30% test datasets
-                    LabeledDatasetTestTrainPair split = new LabeledDatasetTestTrainPair(dataset, 0.3);
-
-                    System.out.println("\n>>> Amount of observations in train dataset " + split.train().rowSize());
-                    System.out.println("\n>>> Amount of observations in test dataset " + split.test().rowSize());
-
-                    LabeledDataset test = split.test();
-                    LabeledDataset train = split.train();
-
-                    System.out.println("\n>>> Create new linear multi-class SVM trainer object.");
-                    Trainer<SVMLinearMultiClassClassificationModel, LabeledDataset> trainer = new SVMLinearMultiClassClassificationTrainer();
-
-                    System.out.println("\n>>> Perform the training to get the model.");
-                    SVMLinearMultiClassClassificationModel mdl = trainer.train(train);
-
-                    System.out.println("\n>>> SVM classification model: " + mdl);
-
-                    // Clone labels
-                    final double[] labels = test.labels();
-
-                    // Save predicted classes to test dataset
-                    LabellingMachine.assignLabels(test, mdl);
-
-                    // Calculate amount of errors on test dataset
-                    int amountOfErrors = 0;
-                    for (int i = 0; i < test.rowSize(); i++) {
-                        if (test.label(i) != labels[i])
-                            amountOfErrors++;
-                    }
-
-                    System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
-                    System.out.println("\n>>> Prediction percentage " + (1 - amountOfErrors / (double) test.rowSize()));
-
-                } catch (IOException e) {
-                    e.printStackTrace();
-                    System.out.println("\n>>> Unexpected exception, check resources: " + e);
-                } finally {
-                    System.out.println("\n>>> SVM Multi-Class classification example completed.");
-                }
-            });
-
-            igniteThread.start();
-            igniteThread.join();
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/examples/src/main/resources/datasets/titanic.txt
----------------------------------------------------------------------
diff --git a/examples/src/main/resources/datasets/titanic.txt b/examples/src/main/resources/datasets/titanic.txt
deleted file mode 100644
index fa2dc70..0000000
--- a/examples/src/main/resources/datasets/titanic.txt
+++ /dev/null
@@ -1,1309 +0,0 @@
-1,1,2,29,0,0,211.3375
-1,1,1,0.9167,1,2,151.55
--1,1,2,2,1,2,151.55
--1,1,1,30,1,2,151.55
--1,1,2,25,1,2,151.55
-1,1,1,48,0,0,26.55
-1,1,2,63,1,0,77.9583
--1,1,1,39,0,0,0
-1,1,2,53,2,0,51.4792
--1,1,1,71,0,0,49.5042
--1,1,1,47,1,0,227.525
-1,1,2,18,1,0,227.525
-1,1,2,24,0,0,69.3
-1,1,2,26,0,0,78.85
-1,1,1,80,0,0,30
--1,1,1,,0,0,25.925
--1,1,1,24,0,1,247.5208
-1,1,2,50,0,1,247.5208
-1,1,2,32,0,0,76.2917
--1,1,1,36,0,0,75.2417
-1,1,1,37,1,1,52.5542
-1,1,2,47,1,1,52.5542
-1,1,1,26,0,0,30
-1,1,2,42,0,0,227.525
-1,1,2,29,0,0,221.7792
--1,1,1,25,0,0,26
-1,1,1,25,1,0,91.0792
-1,1,2,19,1,0,91.0792
-1,1,2,35,0,0,135.6333
-1,1,1,28,0,0,26.55
--1,1,1,45,0,0,35.5
-1,1,1,40,0,0,31
-1,1,2,30,0,0,164.8667
-1,1,2,58,0,0,26.55
--1,1,1,42,0,0,26.55
-1,1,2,45,0,0,262.375
-1,1,2,22,0,1,55
-1,1,1,,0,0,26.55
--1,1,1,41,0,0,30.5
--1,1,1,48,0,0,50.4958
--1,1,1,,0,0,39.6
-1,1,2,44,0,0,27.7208
-1,1,2,59,2,0,51.4792
-1,1,2,60,0,0,76.2917
-1,1,2,41,0,0,134.5
--1,1,1,45,0,0,26.55
--1,1,1,,0,0,31
-1,1,1,42,0,0,26.2875
-1,1,2,53,0,0,27.4458
-1,1,1,36,0,1,512.3292
-1,1,2,58,0,1,512.3292
--1,1,1,33,0,0,5
--1,1,1,28,0,0,47.1
--1,1,1,17,0,0,47.1
-1,1,1,11,1,2,120
-1,1,2,14,1,2,120
-1,1,1,36,1,2,120
-1,1,2,36,1,2,120
--1,1,1,49,0,0,26
-1,1,2,,0,0,27.7208
--1,1,1,36,1,0,78.85
-1,1,2,76,1,0,78.85
--1,1,1,46,1,0,61.175
-1,1,2,47,1,0,61.175
-1,1,1,27,1,0,53.1
-1,1,2,33,1,0,53.1
-1,1,2,36,0,0,262.375
-1,1,2,30,0,0,86.5
-1,1,1,45,0,0,29.7
-1,1,2,,0,1,55
--1,1,1,,0,0,0
--1,1,1,27,1,0,136.7792
-1,1,2,26,1,0,136.7792
-1,1,2,22,0,0,151.55
--1,1,1,,0,0,52
--1,1,1,47,0,0,25.5875
-1,1,2,39,1,1,83.1583
--1,1,1,37,1,1,83.1583
-1,1,2,64,0,2,83.1583
-1,1,2,55,2,0,25.7
--1,1,1,,0,0,26.55
--1,1,1,70,1,1,71
-1,1,2,36,0,2,71
-1,1,2,64,1,1,26.55
--1,1,1,39,1,0,71.2833
-1,1,2,38,1,0,71.2833
-1,1,1,51,0,0,26.55
-1,1,1,27,0,0,30.5
-1,1,2,33,0,0,151.55
--1,1,1,31,1,0,52
-1,1,2,27,1,2,52
-1,1,1,31,1,0,57
-1,1,2,17,1,0,57
-1,1,1,53,1,1,81.8583
-1,1,1,4,0,2,81.8583
-1,1,2,54,1,1,81.8583
--1,1,1,50,1,0,106.425
-1,1,2,27,1,1,247.5208
-1,1,2,48,1,0,106.425
-1,1,2,48,1,0,39.6
-1,1,1,49,1,0,56.9292
--1,1,1,39,0,0,29.7
-1,1,2,23,0,1,83.1583
-1,1,2,38,0,0,227.525
-1,1,2,54,1,0,78.2667
--1,1,2,36,0,0,31.6792
--1,1,1,,0,0,221.7792
-1,1,2,,0,0,31.6833
-1,1,2,,0,0,110.8833
-1,1,1,36,0,0,26.3875
--1,1,1,30,0,0,27.75
-1,1,2,24,3,2,263
-1,1,2,28,3,2,263
-1,1,2,23,3,2,263
--1,1,1,19,3,2,263
--1,1,1,64,1,4,263
-1,1,2,60,1,4,263
-1,1,2,30,0,0,56.9292
--1,1,1,,0,0,26.55
-1,1,1,50,2,0,133.65
-1,1,1,43,1,0,27.7208
-1,1,2,,1,0,133.65
-1,1,2,22,0,2,49.5
-1,1,1,60,1,1,79.2
-1,1,2,48,1,1,79.2
--1,1,1,,0,0,0
--1,1,1,37,1,0,53.1
-1,1,2,35,1,0,53.1
--1,1,1,47,0,0,38.5
-1,1,2,35,0,0,211.5
-1,1,2,22,0,1,59.4
-1,1,2,45,0,1,59.4
--1,1,1,24,0,0,79.2
-1,1,1,49,1,0,89.1042
-1,1,2,,1,0,89.1042
--1,1,1,71,0,0,34.6542
-1,1,1,53,0,0,28.5
-1,1,2,19,0,0,30
--1,1,1,38,0,1,153.4625
-1,1,2,58,0,1,153.4625
-1,1,1,23,0,1,63.3583
-1,1,2,45,0,1,63.3583
--1,1,1,46,0,0,79.2
-1,1,1,25,1,0,55.4417
-1,1,2,25,1,0,55.4417
-1,1,1,48,1,0,76.7292
-1,1,2,49,1,0,76.7292
--1,1,1,,0,0,42.4
--1,1,1,45,1,0,83.475
-1,1,2,35,1,0,83.475
--1,1,1,40,0,0,0
-1,1,1,27,0,0,76.7292
-1,1,1,,0,0,30
-1,1,2,24,0,0,83.1583
--1,1,1,55,1,1,93.5
-1,1,2,52,1,1,93.5
--1,1,1,42,0,0,42.5
--1,1,1,,0,0,51.8625
--1,1,1,55,0,0,50
-1,1,2,16,0,1,57.9792
-1,1,2,44,0,1,57.9792
-1,1,2,51,1,0,77.9583
--1,1,1,42,1,0,52
-1,1,2,35,1,0,52
-1,1,1,35,0,0,26.55
-1,1,1,38,1,0,90
--1,1,1,,0,0,30.6958
-1,1,2,35,1,0,90
-1,1,2,38,0,0,80
--1,1,2,50,0,0,28.7125
-1,1,1,49,0,0,0
--1,1,1,46,0,0,26
--1,1,1,50,0,0,26
--1,1,1,32.5,0,0,211.5
--1,1,1,58,0,0,29.7
--1,1,1,41,1,0,51.8625
-1,1,2,,1,0,51.8625
-1,1,1,42,1,0,52.5542
-1,1,2,45,1,0,52.5542
--1,1,1,,0,0,26.55
-1,1,2,39,0,0,211.3375
-1,1,2,49,0,0,25.9292
-1,1,2,30,0,0,106.425
-1,1,1,35,0,0,512.3292
--1,1,1,,0,0,27.7208
--1,1,1,42,0,0,26.55
-1,1,2,55,0,0,27.7208
-1,1,2,16,0,1,39.4
-1,1,2,51,0,1,39.4
--1,1,1,29,0,0,30
-1,1,2,21,0,0,77.9583
--1,1,1,30,0,0,45.5
-1,1,2,58,0,0,146.5208
-1,1,2,15,0,1,211.3375
--1,1,1,30,0,0,26
-1,1,2,16,0,0,86.5
-1,1,1,,0,0,29.7
--1,1,1,19,1,0,53.1
-1,1,2,18,1,0,53.1
-1,1,2,24,0,0,49.5042
--1,1,1,46,0,0,75.2417
--1,1,1,54,0,0,51.8625
-1,1,1,36,0,0,26.2875
--1,1,1,28,1,0,82.1708
-1,1,2,,1,0,82.1708
--1,1,1,65,0,0,26.55
--1,1,1,44,2,0,90
-1,1,2,33,1,0,90
-1,1,2,37,1,0,90
-1,1,1,30,1,0,57.75
--1,1,1,55,0,0,30.5
--1,1,1,47,0,0,42.4
--1,1,1,37,0,1,29.7
-1,1,2,31,1,0,113.275
-1,1,2,23,1,0,113.275
--1,1,1,58,0,2,113.275
-1,1,2,19,0,2,26.2833
--1,1,1,64,0,0,26
-1,1,2,39,0,0,108.9
-1,1,1,,0,0,25.7417
-1,1,2,22,0,1,61.9792
--1,1,1,65,0,1,61.9792
--1,1,1,28.5,0,0,27.7208
--1,1,1,,0,0,0
--1,1,1,45.5,0,0,28.5
--1,1,1,23,0,0,93.5
--1,1,1,29,1,0,66.6
-1,1,2,22,1,0,66.6
--1,1,1,18,1,0,108.9
-1,1,2,17,1,0,108.9
-1,1,2,30,0,0,93.5
-1,1,1,52,0,0,30.5
--1,1,1,47,0,0,52
-1,1,2,56,0,1,83.1583
--1,1,1,38,0,0,0
-1,1,1,,0,0,39.6
--1,1,1,22,0,0,135.6333
--1,1,1,,0,0,227.525
-1,1,2,43,0,1,211.3375
--1,1,1,31,0,0,50.4958
-1,1,1,45,0,0,26.55
--1,1,1,,0,0,50
-1,1,2,33,0,0,27.7208
--1,1,1,46,0,0,79.2
--1,1,1,36,0,0,40.125
-1,1,2,33,0,0,86.5
--1,1,1,55,1,0,59.4
-1,1,2,54,1,0,59.4
--1,1,1,33,0,0,26.55
-1,1,1,13,2,2,262.375
-1,1,2,18,2,2,262.375
-1,1,2,21,2,2,262.375
--1,1,1,61,1,3,262.375
-1,1,2,48,1,3,262.375
-1,1,1,,0,0,30.5
-1,1,2,24,0,0,69.3
-1,1,1,,0,0,26
-1,1,2,35,1,0,57.75
-1,1,2,30,0,0,31
-1,1,1,34,0,0,26.55
-1,1,2,40,0,0,153.4625
-1,1,1,35,0,0,26.2875
--1,1,1,50,1,0,55.9
-1,1,2,39,1,0,55.9
-1,1,1,56,0,0,35.5
-1,1,1,28,0,0,35.5
--1,1,1,56,0,0,26.55
--1,1,1,56,0,0,30.6958
--1,1,1,24,1,0,60
--1,1,1,,0,0,26
-1,1,2,18,1,0,60
-1,1,1,24,1,0,82.2667
-1,1,2,23,1,0,82.2667
-1,1,1,6,0,2,134.5
-1,1,1,45,1,1,134.5
-1,1,2,40,1,1,134.5
--1,1,1,57,1,0,146.5208
-1,1,2,,1,0,146.5208
-1,1,1,32,0,0,30.5
--1,1,1,62,0,0,26.55
-1,1,1,54,1,0,55.4417
-1,1,2,43,1,0,55.4417
-1,1,2,52,1,0,78.2667
--1,1,1,,0,0,27.7208
-1,1,2,62,0,0,80
--1,1,1,67,1,0,221.7792
--1,1,2,63,1,0,221.7792
--1,1,1,61,0,0,32.3208
-1,1,2,48,0,0,25.9292
-1,1,2,18,0,2,79.65
--1,1,1,52,1,1,79.65
-1,1,2,39,1,1,79.65
-1,1,1,48,1,0,52
-1,1,2,,1,0,52
--1,1,1,49,1,1,110.8833
-1,1,1,17,0,2,110.8833
-1,1,2,39,1,1,110.8833
-1,1,2,,0,0,79.2
-1,1,1,31,0,0,28.5375
--1,1,1,40,0,0,27.7208
--1,1,1,61,0,0,33.5
--1,1,1,47,0,0,34.0208
-1,1,2,35,0,0,512.3292
--1,1,1,64,1,0,75.25
-1,1,2,60,1,0,75.25
--1,1,1,60,0,0,26.55
--1,1,1,54,0,1,77.2875
--1,1,1,21,0,1,77.2875
-1,1,2,55,0,0,135.6333
-1,1,2,31,0,2,164.8667
--1,1,1,57,1,1,164.8667
-1,1,2,45,1,1,164.8667
--1,1,1,50,1,1,211.5
--1,1,1,27,0,2,211.5
-1,1,2,50,1,1,211.5
-1,1,2,21,0,0,26.55
--1,1,1,51,0,1,61.3792
-1,1,1,21,0,1,61.3792
--1,1,1,,0,0,35
-1,1,2,31,0,0,134.5
-1,1,1,,0,0,35.5
--1,1,1,62,0,0,26.55
-1,1,2,36,0,0,135.6333
--1,2,1,30,1,0,24
-1,2,2,28,1,0,24
--1,2,1,30,0,0,13
--1,2,1,18,0,0,11.5
--1,2,1,25,0,0,10.5
--1,2,1,34,1,0,26
-1,2,2,36,1,0,26
--1,2,1,57,0,0,13
--1,2,1,18,0,0,11.5
--1,2,1,23,0,0,10.5
-1,2,2,36,0,0,13
--1,2,1,28,0,0,10.5
--1,2,1,51,0,0,12.525
-1,2,1,32,1,0,26
-1,2,2,19,1,0,26
--1,2,1,28,0,0,26
-1,2,1,1,2,1,39
-1,2,2,4,2,1,39
-1,2,2,12,2,1,39
-1,2,2,36,0,3,39
-1,2,1,34,0,0,13
-1,2,2,19,0,0,13
--1,2,1,23,0,0,13
--1,2,1,26,0,0,13
--1,2,1,42,0,0,13
--1,2,1,27,0,0,13
-1,2,2,24,0,0,13
-1,2,2,15,0,2,39
--1,2,1,60,1,1,39
-1,2,2,40,1,1,39
-1,2,2,20,1,0,26
--1,2,1,25,1,0,26
-1,2,2,36,0,0,13
--1,2,1,25,0,0,13
--1,2,1,42,0,0,13
-1,2,2,42,0,0,13
-1,2,1,0.8333,0,2,29
-1,2,1,26,1,1,29
-1,2,2,22,1,1,29
-1,2,2,35,0,0,21
--1,2,1,,0,0,0
--1,2,1,19,0,0,13
--1,2,2,44,1,0,26
--1,2,1,54,1,0,26
--1,2,1,52,0,0,13.5
--1,2,1,37,1,0,26
--1,2,2,29,1,0,26
-1,2,2,25,1,1,30
-1,2,2,45,0,2,30
--1,2,1,29,1,0,26
-1,2,2,28,1,0,26
--1,2,1,29,0,0,10.5
--1,2,1,28,0,0,13
-1,2,1,24,0,0,10.5
-1,2,2,8,0,2,26.25
--1,2,1,31,1,1,26.25
-1,2,2,31,1,1,26.25
-1,2,2,22,0,0,10.5
--1,2,2,30,0,0,13
--1,2,2,,0,0,21
--1,2,1,21,0,0,11.5
--1,2,1,,0,0,0
-1,2,1,8,1,1,36.75
--1,2,1,18,0,0,73.5
-1,2,2,48,0,2,36.75
-1,2,2,28,0,0,13
--1,2,1,32,0,0,13
--1,2,1,17,0,0,73.5
--1,2,1,29,1,0,27.7208
-1,2,2,24,1,0,27.7208
--1,2,1,25,0,0,31.5
--1,2,1,18,0,0,73.5
-1,2,2,18,0,1,23
-1,2,2,34,0,1,23
--1,2,1,54,0,0,26
-1,2,1,8,0,2,32.5
--1,2,1,42,1,1,32.5
-1,2,2,34,1,1,32.5
-1,2,2,27,1,0,13.8583
-1,2,2,30,1,0,13.8583
--1,2,1,23,0,0,13
--1,2,1,21,0,0,13
--1,2,1,18,0,0,13
--1,2,1,40,1,0,26
-1,2,2,29,1,0,26
--1,2,1,18,0,0,10.5
--1,2,1,36,0,0,13
--1,2,1,,0,0,0
--1,2,2,38,0,0,13
--1,2,1,35,0,0,26
--1,2,1,38,1,0,21
--1,2,1,34,1,0,21
-1,2,2,34,0,0,13
--1,2,1,16,0,0,26
--1,2,1,26,0,0,10.5
--1,2,1,47,0,0,10.5
--1,2,1,21,1,0,11.5
--1,2,1,21,1,0,11.5
--1,2,1,24,0,0,13.5
--1,2,1,24,0,0,13
--1,2,1,34,0,0,13
--1,2,1,30,0,0,13
--1,2,1,52,0,0,13
--1,2,1,30,0,0,13
-1,2,1,0.6667,1,1,14.5
-1,2,2,24,0,2,14.5
--1,2,1,44,0,0,13
-1,2,2,6,0,1,33
--1,2,1,28,0,1,33
-1,2,1,62,0,0,10.5
--1,2,1,30,0,0,10.5
-1,2,2,7,0,2,26.25
--1,2,1,43,1,1,26.25
-1,2,2,45,1,1,26.25
-1,2,2,24,1,2,65
-1,2,2,24,1,2,65
--1,2,1,49,1,2,65
-1,2,2,48,1,2,65
-1,2,2,55,0,0,16
--1,2,1,24,2,0,73.5
--1,2,1,32,2,0,73.5
--1,2,1,21,2,0,73.5
--1,2,2,18,1,1,13
-1,2,2,20,2,1,23
--1,2,1,23,2,1,11.5
--1,2,1,36,0,0,13
-1,2,2,54,1,3,23
--1,2,1,50,0,0,13
--1,2,1,44,1,0,26
-1,2,2,29,1,0,26
--1,2,1,21,0,0,73.5
-1,2,1,42,0,0,13
--1,2,1,63,1,0,26
--1,2,2,60,1,0,26
--1,2,1,33,0,0,12.275
-1,2,2,17,0,0,10.5
--1,2,1,42,1,0,27
-1,2,2,24,2,1,27
--1,2,1,47,0,0,15
--1,2,1,24,2,0,31.5
--1,2,1,22,2,0,31.5
--1,2,1,32,0,0,10.5
-1,2,2,23,0,0,13.7917
--1,2,1,34,1,0,26
-1,2,2,24,1,0,26
--1,2,2,22,0,0,21
-1,2,2,,0,0,12.35
--1,2,1,35,0,0,12.35
-1,2,2,45,0,0,13.5
--1,2,1,57,0,0,12.35
--1,2,1,,0,0,0
--1,2,1,31,0,0,10.5
--1,2,2,26,1,1,26
--1,2,1,30,1,1,26
--1,2,1,,0,0,10.7083
-1,2,2,1,1,2,41.5792
-1,2,2,3,1,2,41.5792
--1,2,1,25,1,2,41.5792
-1,2,2,22,1,2,41.5792
-1,2,2,17,0,0,12
-1,2,2,,0,0,33
-1,2,2,34,0,0,10.5
--1,2,1,36,0,0,12.875
--1,2,1,24,0,0,10.5
--1,2,1,61,0,0,12.35
--1,2,1,50,1,0,26
-1,2,2,42,1,0,26
--1,2,2,57,0,0,10.5
--1,2,1,,0,0,15.0458
-1,2,1,1,0,2,37.0042
--1,2,1,31,1,1,37.0042
-1,2,2,24,1,1,37.0042
--1,2,1,,0,0,15.5792
--1,2,1,30,0,0,13
--1,2,1,40,0,0,16
--1,2,1,32,0,0,13.5
--1,2,1,30,0,0,13
--1,2,1,46,0,0,26
-1,2,2,13,0,1,19.5
-1,2,2,41,0,1,19.5
-1,2,1,19,0,0,10.5
--1,2,1,39,0,0,13
--1,2,1,48,0,0,13
--1,2,1,70,0,0,10.5
--1,2,1,27,0,0,13
--1,2,1,54,0,0,14
--1,2,1,39,0,0,26
--1,2,1,16,0,0,10.5
--1,2,1,62,0,0,9.6875
--1,2,1,32.5,1,0,30.0708
-1,2,2,14,1,0,30.0708
-1,2,1,2,1,1,26
-1,2,1,3,1,1,26
--1,2,1,36.5,0,2,26
--1,2,1,26,0,0,13
--1,2,1,19,1,1,36.75
--1,2,1,28,0,0,13.5
-1,2,1,20,0,0,13.8625
-1,2,2,29,0,0,10.5
--1,2,1,39,0,0,13
-1,2,1,22,0,0,10.5
-1,2,1,,0,0,13.8625
--1,2,1,23,0,0,10.5
-1,2,1,29,0,0,13.8583
--1,2,1,28,0,0,10.5
--1,2,1,,0,0,0
-1,2,2,50,0,1,26
--1,2,1,19,0,0,10.5
--1,2,1,,0,0,15.05
--1,2,1,41,0,0,13
-1,2,2,21,0,1,21
-1,2,2,19,0,0,26
--1,2,1,43,0,1,21
-1,2,2,32,0,0,13
--1,2,1,34,0,0,13
-1,2,1,30,0,0,12.7375
--1,2,1,27,0,0,15.0333
-1,2,2,2,1,1,26
-1,2,2,8,1,1,26
-1,2,2,33,0,2,26
--1,2,1,36,0,0,10.5
--1,2,1,34,1,0,21
-1,2,2,30,3,0,21
-1,2,2,28,0,0,13
--1,2,1,23,0,0,15.0458
-1,2,1,0.8333,1,1,18.75
-1,2,1,3,1,1,18.75
-1,2,2,24,2,3,18.75
-1,2,2,50,0,0,10.5
--1,2,1,19,0,0,10.5
-1,2,2,21,0,0,10.5
--1,2,1,26,0,0,13
--1,2,1,25,0,0,13
--1,2,1,27,0,0,26
-1,2,2,25,0,1,26
-1,2,2,18,0,2,13
-1,2,2,20,0,0,36.75
-1,2,2,30,0,0,13
--1,2,1,59,0,0,13.5
-1,2,2,30,0,0,12.35
--1,2,1,35,0,0,10.5
-1,2,2,40,0,0,13
--1,2,1,25,0,0,13
--1,2,1,41,0,0,15.0458
--1,2,1,25,0,0,10.5
--1,2,1,18.5,0,0,13
--1,2,1,14,0,0,65
-1,2,2,50,0,0,10.5
--1,2,1,23,0,0,13
-1,2,2,28,0,0,12.65
-1,2,2,27,0,0,10.5
--1,2,1,29,1,0,21
--1,2,2,27,1,0,21
--1,2,1,40,0,0,13
-1,2,2,31,0,0,21
--1,2,1,30,1,0,21
--1,2,1,23,1,0,10.5
-1,2,2,31,0,0,21
--1,2,1,,0,0,0
-1,2,2,12,0,0,15.75
-1,2,2,40,0,0,15.75
-1,2,2,32.5,0,0,13
--1,2,1,27,1,0,26
-1,2,2,29,1,0,26
-1,2,1,2,1,1,23
-1,2,2,4,1,1,23
-1,2,2,29,0,2,23
-1,2,2,0.9167,1,2,27.75
-1,2,2,5,1,2,27.75
--1,2,1,36,1,2,27.75
-1,2,2,33,1,2,27.75
--1,2,1,66,0,0,10.5
--1,2,1,,0,0,12.875
-1,2,1,31,0,0,13
-1,2,1,,0,0,13
-1,2,2,26,0,0,13.5
--1,2,2,24,0,0,13
--1,3,1,42,0,0,7.55
--1,3,1,13,0,2,20.25
--1,3,1,16,1,1,20.25
-1,3,2,35,1,1,20.25
-1,3,2,16,0,0,7.65
-1,3,1,25,0,0,7.65
-1,3,1,20,0,0,7.925
-1,3,2,18,0,0,7.2292
--1,3,1,30,0,0,7.25
--1,3,1,26,0,0,8.05
--1,3,2,40,1,0,9.475
-1,3,1,0.8333,0,1,9.35
-1,3,2,18,0,1,9.35
-1,3,1,26,0,0,18.7875
--1,3,1,26,0,0,7.8875
--1,3,1,20,0,0,7.925
--1,3,1,24,0,0,7.05
--1,3,1,25,0,0,7.05
--1,3,1,35,0,0,8.05
--1,3,1,18,0,0,8.3
--1,3,1,32,0,0,22.525
-1,3,2,19,1,0,7.8542
--1,3,1,4,4,2,31.275
--1,3,2,6,4,2,31.275
--1,3,2,2,4,2,31.275
-1,3,2,17,4,2,7.925
--1,3,2,38,4,2,7.775
--1,3,2,9,4,2,31.275
--1,3,2,11,4,2,31.275
--1,3,1,39,1,5,31.275
-1,3,1,27,0,0,7.7958
--1,3,1,26,0,0,7.775
--1,3,2,39,1,5,31.275
--1,3,1,20,0,0,7.8542
--1,3,1,26,0,0,7.8958
--1,3,1,25,1,0,17.8
--1,3,2,18,1,0,17.8
--1,3,1,24,0,0,7.775
--1,3,1,35,0,0,7.05
--1,3,1,5,4,2,31.3875
--1,3,1,9,4,2,31.3875
-1,3,1,3,4,2,31.3875
--1,3,1,13,4,2,31.3875
-1,3,2,5,4,2,31.3875
--1,3,1,40,1,5,31.3875
-1,3,1,23,0,0,7.7958
-1,3,2,38,1,5,31.3875
-1,3,2,45,0,0,7.225
--1,3,1,21,0,0,7.225
--1,3,1,23,0,0,7.05
--1,3,2,17,0,0,14.4583
--1,3,1,30,0,0,7.225
--1,3,1,23,0,0,7.8542
-1,3,2,13,0,0,7.2292
--1,3,1,20,0,0,7.225
--1,3,1,32,1,0,15.85
-1,3,2,33,3,0,15.85
-1,3,2,0.75,2,1,19.2583
-1,3,2,0.75,2,1,19.2583
-1,3,2,5,2,1,19.2583
-1,3,2,24,0,3,19.2583
-1,3,2,18,0,0,8.05
--1,3,1,40,0,0,7.225
--1,3,1,26,0,0,7.8958
-1,3,1,20,0,0,7.2292
--1,3,2,18,0,1,14.4542
--1,3,2,45,0,1,14.4542
--1,3,2,27,0,0,7.8792
--1,3,1,22,0,0,8.05
--1,3,1,19,0,0,8.05
--1,3,1,26,0,0,7.775
--1,3,1,22,0,0,9.35
--1,3,1,,0,0,7.2292
--1,3,1,20,0,0,4.0125
-1,3,1,32,0,0,56.4958
--1,3,1,21,0,0,7.775
--1,3,1,18,0,0,7.75
--1,3,1,26,0,0,7.8958
--1,3,1,6,1,1,15.2458
--1,3,2,9,1,1,15.2458
--1,3,1,,0,0,7.225
--1,3,2,,0,2,15.2458
--1,3,2,,0,2,7.75
--1,3,1,40,1,1,15.5
--1,3,2,32,1,1,15.5
--1,3,1,21,0,0,16.1
-1,3,2,22,0,0,7.725
--1,3,2,20,0,0,7.8542
--1,3,1,29,1,0,7.0458
--1,3,1,22,1,0,7.25
--1,3,1,22,0,0,7.7958
--1,3,1,35,0,0,8.05
--1,3,2,18.5,0,0,7.2833
-1,3,1,21,0,0,7.8208
--1,3,1,19,0,0,6.75
--1,3,2,18,0,0,7.8792
--1,3,2,21,0,0,8.6625
--1,3,2,30,0,0,8.6625
--1,3,1,18,0,0,8.6625
--1,3,1,38,0,0,8.6625
--1,3,1,17,0,0,8.6625
--1,3,1,17,0,0,8.6625
--1,3,2,21,0,0,7.75
--1,3,1,21,0,0,7.75
--1,3,1,21,0,0,8.05
--1,3,1,,1,0,14.4583
--1,3,2,,1,0,14.4583
--1,3,1,28,0,0,7.7958
--1,3,1,24,0,0,7.8542
-1,3,2,16,0,0,7.75
--1,3,2,37,0,0,7.75
--1,3,1,28,0,0,7.25
--1,3,1,24,0,0,8.05
--1,3,1,21,0,0,7.7333
-1,3,1,32,0,0,56.4958
--1,3,1,29,0,0,8.05
--1,3,1,26,1,0,14.4542
--1,3,1,18,1,0,14.4542
--1,3,1,20,0,0,7.05
-1,3,1,18,0,0,8.05
--1,3,1,24,0,0,7.25
--1,3,1,36,0,0,7.4958
--1,3,1,24,0,0,7.4958
--1,3,1,31,0,0,7.7333
--1,3,1,31,0,0,7.75
-1,3,2,22,0,0,7.75
--1,3,2,30,0,0,7.6292
--1,3,1,70.5,0,0,7.75
--1,3,1,43,0,0,8.05
--1,3,1,35,0,0,7.8958
--1,3,1,27,0,0,7.8958
--1,3,1,19,0,0,7.8958
--1,3,1,30,0,0,8.05
-1,3,1,9,1,1,15.9
-1,3,1,3,1,1,15.9
-1,3,2,36,0,2,15.9
--1,3,1,59,0,0,7.25
--1,3,1,19,0,0,8.1583
-1,3,2,17,0,1,16.1
--1,3,1,44,0,1,16.1
--1,3,1,17,0,0,8.6625
--1,3,1,22.5,0,0,7.225
-1,3,1,45,0,0,8.05
--1,3,2,22,0,0,10.5167
--1,3,1,19,0,0,10.1708
-1,3,2,30,0,0,6.95
-1,3,1,29,0,0,7.75
--1,3,1,0.3333,0,2,14.4
--1,3,1,34,1,1,14.4
--1,3,2,28,1,1,14.4
--1,3,1,27,0,0,7.8958
--1,3,1,25,0,0,7.8958
--1,3,1,24,2,0,24.15
--1,3,1,22,0,0,8.05
--1,3,1,21,2,0,24.15
--1,3,1,17,2,0,8.05
--1,3,1,,1,0,16.1
-1,3,2,,1,0,16.1
-1,3,1,36.5,1,0,17.4
-1,3,2,36,1,0,17.4
-1,3,1,30,0,0,9.5
--1,3,1,16,0,0,9.5
-1,3,1,1,1,2,20.575
-1,3,2,0.1667,1,2,20.575
--1,3,1,26,1,2,20.575
-1,3,2,33,1,2,20.575
--1,3,1,25,0,0,7.8958
--1,3,1,,0,0,7.8958
--1,3,1,,0,0,7.8958
--1,3,1,22,0,0,7.25
--1,3,1,36,0,0,7.25
-1,3,2,19,0,0,7.8792
--1,3,1,17,0,0,7.8958
--1,3,1,42,0,0,8.6625
--1,3,1,43,0,0,7.8958
--1,3,1,,0,0,7.2292
--1,3,1,32,0,0,7.75
-1,3,1,19,0,0,8.05
-1,3,2,30,0,0,12.475
--1,3,2,24,0,0,7.75
-1,3,2,23,0,0,8.05
--1,3,1,33,0,0,7.8958
--1,3,1,65,0,0,7.75
-1,3,1,24,0,0,7.55
--1,3,1,23,1,0,13.9
-1,3,2,22,1,0,13.9
--1,3,1,18,0,0,7.775
--1,3,1,16,0,0,7.775
--1,3,1,45,0,0,6.975
--1,3,1,,0,0,7.225
--1,3,1,39,0,2,7.2292
--1,3,1,17,1,1,7.2292
--1,3,1,15,1,1,7.2292
--1,3,1,47,0,0,7.25
-1,3,2,5,0,0,12.475
--1,3,1,,0,0,7.225
--1,3,1,40.5,0,0,15.1
--1,3,1,40.5,0,0,7.75
-1,3,1,,0,0,7.05
--1,3,1,18,0,0,7.7958
--1,3,2,,0,0,7.75
--1,3,1,,0,0,7.75
--1,3,1,,0,0,6.95
--1,3,1,26,0,0,7.8792
--1,3,1,,0,0,7.75
-1,3,1,,0,0,56.4958
--1,3,2,21,2,2,34.375
--1,3,2,9,2,2,34.375
--1,3,1,,0,0,8.05
--1,3,1,18,2,2,34.375
--1,3,1,16,1,3,34.375
--1,3,2,48,1,3,34.375
--1,3,1,,0,0,7.75
--1,3,1,,0,0,7.25
--1,3,1,25,0,0,7.7417
--1,3,1,,0,0,14.5
--1,3,1,,0,0,7.8958
--1,3,1,22,0,0,8.05
-1,3,2,16,0,0,7.7333
-1,3,2,,0,0,7.75
-1,3,1,9,0,2,20.525
--1,3,1,33,1,1,20.525
--1,3,1,41,0,0,7.85
-1,3,2,31,1,1,20.525
--1,3,1,38,0,0,7.05
--1,3,1,9,5,2,46.9
--1,3,1,1,5,2,46.9
--1,3,1,11,5,2,46.9
--1,3,2,10,5,2,46.9
--1,3,2,16,5,2,46.9
--1,3,1,14,5,2,46.9
--1,3,1,40,1,6,46.9
--1,3,2,43,1,6,46.9
--1,3,1,51,0,0,8.05
--1,3,1,32,0,0,8.3625
--1,3,1,,0,0,8.05
--1,3,1,20,0,0,9.8458
--1,3,1,37,2,0,7.925
--1,3,1,28,2,0,7.925
--1,3,1,19,0,0,7.775
--1,3,2,24,0,0,8.85
--1,3,2,17,0,0,7.7333
--1,3,1,,1,0,19.9667
--1,3,1,,1,0,19.9667
--1,3,1,28,1,0,15.85
-1,3,2,24,1,0,15.85
--1,3,1,20,0,0,9.5
--1,3,1,23.5,0,0,7.2292
--1,3,1,41,2,0,14.1083
--1,3,1,26,1,0,7.8542
--1,3,1,21,0,0,7.8542
-1,3,2,45,1,0,14.1083
--1,3,2,,0,0,7.55
--1,3,1,25,0,0,7.25
--1,3,1,,0,0,6.8583
--1,3,1,11,0,0,18.7875
-1,3,2,,0,0,7.75
-1,3,1,27,0,0,6.975
-1,3,1,,0,0,56.4958
--1,3,2,18,0,0,6.75
-1,3,2,26,0,0,7.925
--1,3,2,23,0,0,7.925
-1,3,2,22,0,0,8.9625
--1,3,1,28,0,0,7.8958
--1,3,2,28,0,0,7.775
--1,3,2,,0,0,7.75
-1,3,2,2,0,1,12.2875
-1,3,2,22,1,1,12.2875
--1,3,1,43,0,0,6.45
--1,3,1,28,0,0,22.525
-1,3,2,27,0,0,7.925
--1,3,1,,0,0,7.75
-1,3,2,,0,0,8.05
--1,3,1,42,0,0,7.65
-1,3,1,,0,0,7.8875
--1,3,1,30,0,0,7.2292
--1,3,1,,0,0,7.8958
--1,3,2,27,1,0,7.925
--1,3,2,25,1,0,7.925
--1,3,1,,0,0,7.8958
-1,3,1,29,0,0,7.8958
-1,3,1,21,0,0,7.7958
--1,3,1,,0,0,7.05
--1,3,1,20,0,0,7.8542
--1,3,1,48,0,0,7.8542
--1,3,1,17,1,0,7.0542
-1,3,2,,0,0,7.75
-1,3,1,,0,0,8.1125
--1,3,1,34,0,0,6.4958
-1,3,1,26,0,0,7.775
--1,3,1,22,0,0,7.7958
--1,3,1,33,0,0,8.6542
--1,3,1,31,0,0,7.775
--1,3,1,29,0,0,7.8542
-1,3,1,4,1,1,11.1333
-1,3,2,1,1,1,11.1333
--1,3,1,49,0,0,0
--1,3,1,33,0,0,7.775
--1,3,1,19,0,0,0
-1,3,2,27,0,2,11.1333
--1,3,1,,1,2,23.45
--1,3,2,,1,2,23.45
--1,3,1,,1,2,23.45
--1,3,2,,1,2,23.45
--1,3,1,23,0,0,7.8958
-1,3,1,32,0,0,7.8542
--1,3,1,27,0,0,7.8542
--1,3,2,20,1,0,9.825
--1,3,2,21,1,0,9.825
-1,3,1,32,0,0,7.925
--1,3,1,17,0,0,7.125
--1,3,1,21,0,0,8.4333
--1,3,1,30,0,0,7.8958
-1,3,1,21,0,0,7.7958
--1,3,1,33,0,0,7.8542
--1,3,1,22,0,0,7.5208
-1,3,2,4,0,1,13.4167
-1,3,1,39,0,1,13.4167
--1,3,1,,0,0,7.2292
--1,3,1,18.5,0,0,7.2292
--1,3,1,,0,0,7.75
--1,3,1,,0,0,7.25
-1,3,2,,0,0,7.75
-1,3,2,,0,0,7.75
--1,3,1,34.5,0,0,7.8292
--1,3,1,44,0,0,8.05
-1,3,1,,0,0,7.75
--1,3,1,,1,0,14.4542
--1,3,2,,1,0,14.4542
--1,3,1,,1,0,7.75
--1,3,1,,1,0,7.75
--1,3,1,,0,0,7.7375
--1,3,2,22,2,0,8.6625
--1,3,1,26,2,0,8.6625
-1,3,2,4,0,2,22.025
-1,3,1,29,3,1,22.025
-1,3,2,26,1,1,22.025
--1,3,2,1,1,1,12.1833
--1,3,1,18,1,1,7.8542
--1,3,2,36,0,2,12.1833
--1,3,1,,0,0,7.8958
-1,3,1,25,0,0,7.2292
--1,3,1,,0,0,7.225
--1,3,2,37,0,0,9.5875
--1,3,1,,0,0,7.8958
-1,3,1,,0,0,56.4958
--1,3,1,,0,0,56.4958
-1,3,2,22,0,0,7.25
--1,3,1,,0,0,7.75
-1,3,1,26,0,0,56.4958
--1,3,1,29,0,0,9.4833
--1,3,1,29,0,0,7.775
--1,3,1,22,0,0,7.775
-1,3,1,22,0,0,7.225
--1,3,1,,3,1,25.4667
--1,3,2,,3,1,25.4667
--1,3,2,,3,1,25.4667
--1,3,2,,3,1,25.4667
--1,3,2,,0,4,25.4667
--1,3,1,32,0,0,7.925
--1,3,1,34.5,0,0,6.4375
--1,3,2,,1,0,15.5
--1,3,1,,1,0,15.5
--1,3,1,36,0,0,0
--1,3,1,39,0,0,24.15
--1,3,1,24,0,0,9.5
--1,3,2,25,0,0,7.775
--1,3,2,45,0,0,7.75
--1,3,1,36,1,0,15.55
--1,3,2,30,1,0,15.55
-1,3,1,20,1,0,7.925
--1,3,1,,0,0,7.8792
--1,3,1,28,0,0,56.4958
--1,3,1,,0,0,7.55
--1,3,1,30,1,0,16.1
--1,3,2,26,1,0,16.1
--1,3,1,,0,0,7.8792
--1,3,1,20.5,0,0,7.25
-1,3,1,27,0,0,8.6625
--1,3,1,51,0,0,7.0542
-1,3,2,23,0,0,7.8542
-1,3,1,32,0,0,7.5792
--1,3,1,,0,0,7.8958
--1,3,1,,0,0,7.55
-1,3,2,,0,0,7.75
-1,3,1,24,0,0,7.1417
--1,3,1,22,0,0,7.125
--1,3,2,,0,0,7.8792
--1,3,1,,0,0,7.75
--1,3,1,,0,0,8.05
--1,3,1,29,0,0,7.925
-1,3,1,,0,0,7.2292
--1,3,2,30.5,0,0,7.75
-1,3,2,,0,0,7.7375
--1,3,1,,0,0,7.2292
--1,3,1,35,0,0,7.8958
--1,3,1,33,0,0,7.8958
-1,3,2,,0,0,7.225
--1,3,1,,0,0,7.8958
-1,3,2,,0,0,7.75
-1,3,1,,0,0,7.75
-1,3,2,,2,0,23.25
-1,3,2,,2,0,23.25
-1,3,1,,2,0,23.25
-1,3,2,,0,0,7.7875
--1,3,1,,0,0,15.5
-1,3,2,,0,0,7.8792
-1,3,2,15,0,0,8.0292
--1,3,2,35,0,0,7.75
--1,3,1,,0,0,7.75
--1,3,1,24,1,0,16.1
--1,3,2,19,1,0,16.1
--1,3,2,,0,0,7.75
--1,3,2,,0,0,8.05
--1,3,2,,0,0,8.05
--1,3,1,55.5,0,0,8.05
--1,3,1,,0,0,7.75
-1,3,1,21,0,0,7.775
--1,3,1,,0,0,8.05
--1,3,1,24,0,0,7.8958
--1,3,1,21,0,0,7.8958
--1,3,1,28,0,0,7.8958
--1,3,1,,0,0,7.8958
-1,3,2,,0,0,7.8792
--1,3,1,25,0,0,7.65
-1,3,1,6,0,1,12.475
-1,3,2,27,0,1,12.475
--1,3,1,,0,0,8.05
-1,3,2,,1,0,24.15
--1,3,1,,1,0,24.15
--1,3,1,,0,0,8.4583
--1,3,1,34,0,0,8.05
--1,3,1,,0,0,7.75
-1,3,1,,0,0,7.775
-1,3,1,,1,1,15.2458
-1,3,1,,1,1,15.2458
-1,3,2,,0,2,15.2458
-1,3,2,,0,0,7.2292
--1,3,1,,0,0,8.05
-1,3,2,,0,0,7.7333
-1,3,2,24,0,0,7.75
--1,3,1,,0,0,8.05
-1,3,2,,1,0,15.5
-1,3,2,,1,0,15.5
-1,3,2,,0,0,15.5
--1,3,1,18,0,0,7.75
--1,3,1,22,0,0,7.8958
-1,3,2,15,0,0,7.225
-1,3,2,1,0,2,15.7417
-1,3,1,20,1,1,15.7417
-1,3,2,19,1,1,15.7417
--1,3,1,33,0,0,8.05
--1,3,1,,0,0,7.8958
--1,3,1,,0,0,7.2292
--1,3,2,,0,0,7.75
--1,3,1,,0,0,7.8958
-1,3,1,12,1,0,11.2417
-1,3,2,14,1,0,11.2417
--1,3,2,29,0,0,7.925
--1,3,1,28,0,0,8.05
-1,3,2,18,0,0,7.775
-1,3,2,26,0,0,7.8542
--1,3,1,21,0,0,7.8542
--1,3,1,41,0,0,7.125
-1,3,1,39,0,0,7.925
--1,3,1,21,0,0,7.8
--1,3,1,28.5,0,0,7.2292
-1,3,2,22,0,0,7.75
--1,3,1,61,0,0,6.2375
--1,3,1,,1,0,15.5
--1,3,1,,0,0,7.8292
-1,3,2,,1,0,15.5
--1,3,1,,0,0,7.7333
--1,3,1,,0,0,7.75
--1,3,1,,0,0,7.75
--1,3,1,23,0,0,9.225
--1,3,2,,0,0,7.75
-1,3,2,,0,0,7.75
-1,3,2,,0,0,7.8792
-1,3,2,22,0,0,7.775
-1,3,1,,0,0,7.75
-1,3,2,,0,0,7.8292
-1,3,1,9,0,1,3.1708
--1,3,1,28,0,0,22.525
--1,3,1,42,0,1,8.4042
--1,3,1,,0,0,7.3125
--1,3,2,31,0,0,7.8542
--1,3,1,28,0,0,7.8542
-1,3,1,32,0,0,7.775
--1,3,1,20,0,0,9.225
--1,3,2,23,0,0,8.6625
--1,3,2,20,0,0,8.6625
--1,3,1,20,0,0,8.6625
--1,3,1,16,0,0,9.2167
-1,3,2,31,0,0,8.6833
--1,3,2,,0,0,7.6292
--1,3,1,2,3,1,21.075
--1,3,1,6,3,1,21.075
--1,3,2,3,3,1,21.075
--1,3,2,8,3,1,21.075
--1,3,2,29,0,4,21.075
--1,3,1,1,4,1,39.6875
--1,3,1,7,4,1,39.6875
--1,3,1,2,4,1,39.6875
--1,3,1,16,4,1,39.6875
--1,3,1,14,4,1,39.6875
--1,3,2,41,0,5,39.6875
--1,3,1,21,0,0,8.6625
--1,3,1,19,0,0,14.5
--1,3,1,,0,0,8.7125
--1,3,1,32,0,0,7.8958
--1,3,1,0.75,1,1,13.775
--1,3,2,3,1,1,13.775
--1,3,2,26,0,2,13.775
--1,3,1,,0,0,7
--1,3,1,,0,0,7.775
--1,3,1,,0,0,8.05
--1,3,1,21,0,0,7.925
--1,3,1,25,0,0,7.925
--1,3,1,22,0,0,7.25
-1,3,1,25,1,0,7.775
-1,3,1,,1,1,22.3583
-1,3,2,,1,1,22.3583
-1,3,2,,0,2,22.3583
--1,3,2,,0,0,8.1375
--1,3,1,24,0,0,8.05
--1,3,2,28,0,0,7.8958
--1,3,1,19,0,0,7.8958
--1,3,1,,0,0,7.8958
--1,3,1,25,1,0,7.775
--1,3,2,18,0,0,7.775
-1,3,1,32,0,0,8.05
--1,3,1,,0,0,7.8958
--1,3,1,17,0,0,8.6625
--1,3,1,24,0,0,8.6625
--1,3,1,,0,0,7.8958
--1,3,2,,0,0,8.1125
--1,3,1,,0,0,7.2292
--1,3,1,,0,0,7.25
--1,3,1,38,0,0,7.8958
--1,3,1,21,0,0,8.05
--1,3,1,10,4,1,29.125
--1,3,1,4,4,1,29.125
--1,3,1,7,4,1,29.125
--1,3,1,2,4,1,29.125
--1,3,1,8,4,1,29.125
--1,3,2,39,0,5,29.125
--1,3,2,22,0,0,39.6875
--1,3,1,35,0,0,7.125
-1,3,2,,0,0,7.7208
--1,3,1,,0,0,14.5
--1,3,2,,0,0,14.5
--1,3,1,50,1,0,14.5
--1,3,2,47,1,0,14.5
--1,3,1,,0,0,8.05
--1,3,1,,0,0,7.775
--1,3,2,2,1,1,20.2125
--1,3,1,18,1,1,20.2125
--1,3,2,41,0,2,20.2125
-1,3,2,,0,0,8.05
--1,3,1,50,0,0,8.05
--1,3,1,16,0,0,8.05
-1,3,1,,0,0,7.75
--1,3,1,,0,0,24.15
--1,3,1,,0,0,7.2292
--1,3,1,25,0,0,7.225
--1,3,1,,0,0,7.225
--1,3,1,,0,0,7.7292
--1,3,1,,0,0,7.575
--1,3,1,38.5,0,0,7.25
--1,3,1,,8,2,69.55
--1,3,1,14.5,8,2,69.55
--1,3,2,,8,2,69.55
--1,3,2,,8,2,69.55
--1,3,2,,8,2,69.55
--1,3,2,,8,2,69.55
--1,3,1,,8,2,69.55
--1,3,1,,8,2,69.55
--1,3,1,,8,2,69.55
--1,3,1,,1,9,69.55
--1,3,2,,1,9,69.55
--1,3,1,24,0,0,9.325
-1,3,2,21,0,0,7.65
--1,3,1,39,0,0,7.925
--1,3,1,,2,0,21.6792
--1,3,1,,2,0,21.6792
--1,3,1,,2,0,21.6792
-1,3,2,1,1,1,16.7
-1,3,2,24,0,2,16.7
-1,3,2,4,1,1,16.7
-1,3,1,25,0,0,9.5
--1,3,1,20,0,0,8.05
--1,3,1,24.5,0,0,8.05
--1,3,1,,0,0,7.725
--1,3,1,,0,0,7.8958
--1,3,1,,0,0,7.75
-1,3,1,29,0,0,9.5
--1,3,1,,0,0,15.1
-1,3,2,,0,0,7.7792
--1,3,1,,0,0,8.05
--1,3,1,,0,0,8.05
--1,3,1,22,0,0,7.2292
--1,3,1,,0,0,8.05
--1,3,1,40,0,0,7.8958
--1,3,1,21,0,0,7.925
-1,3,2,18,0,0,7.4958
--1,3,1,4,3,2,27.9
--1,3,1,10,3,2,27.9
--1,3,2,9,3,2,27.9
--1,3,2,2,3,2,27.9
--1,3,1,40,1,4,27.9
--1,3,2,45,1,4,27.9
--1,3,1,,0,0,7.8958
--1,3,1,,0,0,8.05
--1,3,1,,0,0,8.6625
--1,3,1,,0,0,7.75
-1,3,2,,0,0,7.7333
--1,3,1,19,0,0,7.65
--1,3,1,30,0,0,8.05
--1,3,1,,0,0,8.05
--1,3,1,32,0,0,8.05
--1,3,1,,0,0,7.8958
--1,3,1,33,0,0,8.6625
-1,3,2,23,0,0,7.55
--1,3,1,21,0,0,8.05
--1,3,1,60.5,0,0,0
--1,3,1,19,0,0,7.8958
--1,3,2,22,0,0,9.8375
-1,3,1,31,0,0,7.925
--1,3,1,27,0,0,8.6625
--1,3,2,2,0,1,10.4625
--1,3,2,29,1,1,10.4625
-1,3,1,16,0,0,8.05
-1,3,1,44,0,0,7.925
--1,3,1,25,0,0,7.05
--1,3,1,74,0,0,7.775
-1,3,1,14,0,0,9.225
--1,3,1,24,0,0,7.7958
-1,3,1,25,0,0,7.7958
--1,3,1,34,0,0,8.05
-1,3,1,0.4167,0,1,8.5167
--1,3,1,,1,0,6.4375
--1,3,1,,0,0,6.4375
--1,3,1,,0,0,7.225
-1,3,2,16,1,1,8.5167
--1,3,1,,0,0,8.05
--1,3,1,,1,0,16.1
-1,3,2,,1,0,16.1
--1,3,1,32,0,0,7.925
--1,3,1,,0,0,7.75
--1,3,1,,0,0,7.8958
--1,3,1,30.5,0,0,8.05
--1,3,1,44,0,0,8.05
--1,3,1,,0,0,7.2292
-1,3,1,25,0,0,0
--1,3,1,,0,0,7.2292
-1,3,1,7,1,1,15.2458
-1,3,2,9,1,1,15.2458
-1,3,2,29,0,2,15.2458
--1,3,1,36,0,0,7.8958
-1,3,2,18,0,0,9.8417
-1,3,2,63,0,0,9.5875
--1,3,1,,1,1,14.5
--1,3,1,11.5,1,1,14.5
--1,3,1,40.5,0,2,14.5
--1,3,2,10,0,2,24.15
--1,3,1,36,1,1,24.15
--1,3,2,30,1,1,24.15
--1,3,1,,0,0,9.5
--1,3,1,33,0,0,9.5
--1,3,1,28,0,0,9.5
--1,3,1,28,0,0,9.5
--1,3,1,47,0,0,9
--1,3,2,18,2,0,18
--1,3,1,31,3,0,18
--1,3,1,16,2,0,18
--1,3,2,31,1,0,18
-1,3,1,22,0,0,7.225
--1,3,1,20,0,0,7.8542
--1,3,2,14,0,0,7.8542
--1,3,1,22,0,0,7.8958
--1,3,1,22,0,0,9
--1,3,1,,0,0,8.05
--1,3,1,,0,0,7.55
--1,3,1,,0,0,8.05
--1,3,1,32.5,0,0,9.5
-1,3,2,38,0,0,7.2292
--1,3,1,51,0,0,7.75
--1,3,1,18,1,0,6.4958
--1,3,1,21,1,0,6.4958
-1,3,2,47,1,0,7
--1,3,1,,0,0,8.7125
--1,3,1,,0,0,7.55
--1,3,1,,0,0,8.05
--1,3,1,28.5,0,0,16.1
--1,3,1,21,0,0,7.25
--1,3,1,27,0,0,8.6625
--1,3,1,,0,0,7.25
--1,3,1,36,0,0,9.5
--1,3,1,27,1,0,14.4542
-1,3,2,15,1,0,14.4542
--1,3,1,45.5,0,0,7.225
--1,3,1,,0,0,7.225
--1,3,1,,0,0,14.4583
--1,3,2,14.5,1,0,14.4542
--1,3,2,,1,0,14.4542
--1,3,1,26.5,0,0,7.225
--1,3,1,27,0,0,7.225
--1,3,1,29,0,0,7.875

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/structures/Dataset.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/structures/Dataset.java b/modules/ml/src/main/java/org/apache/ignite/ml/structures/Dataset.java
index 89ff1b4..cbed297 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/structures/Dataset.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/structures/Dataset.java
@@ -86,6 +86,16 @@ public class Dataset<Row extends DatasetRow> implements Serializable, Externaliz
     }
 
     /**
+     * Creates new Dataset by given data.
+     *
+     * @param data Should be initialized with one vector at least.
+     */
+    public Dataset(Row[] data) {
+        this.data = data;
+        this.rowSize = data.length;
+    }
+
+    /**
      * Creates new Dataset and initialized with empty data structure.
      *
      * @param rowSize Amount of instances. Should be > 0.

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java b/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java
index 0028a16..3239116 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java
@@ -27,7 +27,7 @@ import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector;
 /**
  * Class for set of labeled vectors.
  */
-public class LabeledDataset<L, Row extends LabeledVector> extends Dataset<Row> {
+public class LabeledDataset<L, Row extends LabeledVector> extends Dataset<Row> implements AutoCloseable {
     /**
      * Default constructor (required by Externalizable).
      */
@@ -70,6 +70,15 @@ public class LabeledDataset<L, Row extends LabeledVector> extends Dataset<Row> {
         initializeDataWithLabeledVectors();
     }
 
+    /**
+     * Creates new Labeled Dataset by given data.
+     *
+     * @param data Should be initialized with one vector at least.
+     */
+    public LabeledDataset(Row[] data) {
+        super(data);
+    }
+
     /** */
     private void initializeDataWithLabeledVectors() {
         data = (Row[])new LabeledVector[rowSize];
@@ -207,4 +216,9 @@ public class LabeledDataset<L, Row extends LabeledVector> extends Dataset<Row> {
 
         return res;
     }
+
+    /** Closes LabeledDataset. */
+    @Override public void close() throws Exception {
+
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java
index 6d93402..4fd2e0e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java
@@ -146,7 +146,9 @@ public class SVMLinearBinaryClassificationModel implements Model<Vector, Double>
             return true;
         if (o == null || getClass() != o.getClass())
             return false;
+
         SVMLinearBinaryClassificationModel mdl = (SVMLinearBinaryClassificationModel)o;
+
         return Double.compare(mdl.intercept, intercept) == 0
             && Double.compare(mdl.threshold, threshold) == 0
             && Boolean.compare(mdl.isKeepingRawLabels, isKeepingRawLabels) == 0
@@ -176,7 +178,7 @@ public class SVMLinearBinaryClassificationModel implements Model<Vector, Double>
             return builder.toString();
         }
 
-        return "LinearRegressionModel{" +
+        return "SVMModel{" +
             "weights=" + weights +
             ", intercept=" + intercept +
             '}';

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
index ee3b6e8..e745ca9 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
@@ -18,10 +18,13 @@
 package org.apache.ignite.ml.svm;
 
 import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.ml.Trainer;
+import org.apache.ignite.ml.DatasetTrainer;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.PartitionDataBuilder;
 import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector;
 import org.apache.ignite.ml.structures.LabeledDataset;
 import org.apache.ignite.ml.structures.LabeledVector;
 import org.jetbrains.annotations.NotNull;
@@ -32,33 +35,50 @@ import org.jetbrains.annotations.NotNull;
  * and +1 labels for two classes and makes binary classification. </p> The paper about this algorithm could be found
  * here https://arxiv.org/abs/1409.1458.
  */
-public class SVMLinearBinaryClassificationTrainer implements Trainer<SVMLinearBinaryClassificationModel, LabeledDataset> {
+public class SVMLinearBinaryClassificationTrainer<K, V>
+    implements DatasetTrainer<K, V, SVMLinearBinaryClassificationModel> {
     /** Amount of outer SDCA algorithm iterations. */
-    private int amountOfIterations = 20;
+    private int amountOfIterations = 200;
 
     /** Amount of local SDCA algorithm iterations. */
-    private int amountOfLocIterations = 50;
+    private int amountOfLocIterations = 100;
 
     /** Regularization parameter. */
-    private double lambda = 0.2;
+    private double lambda = 0.4;
 
-    /** This flag enables distributed mode for this algorithm. */
-    private boolean isDistributed;
+    /** Dataset. */
+    private Dataset<SVMPartitionContext, LabeledDataset<Double, LabeledVector>> dataset;
 
     /**
-     * Returns model based on data
+     * Trains model based on the specified data.
      *
-     * @param data data to build model
-     * @return model
+     * @param datasetBuilder   Dataset builder.
+     * @param featureExtractor Feature extractor.
+     * @param lbExtractor      Label extractor.
+     * @param cols             Number of columns.
+     * @return Model.
      */
-    @Override public SVMLinearBinaryClassificationModel train(LabeledDataset data) {
-        isDistributed = data.isDistributed();
+    @Override public SVMLinearBinaryClassificationModel fit(DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor, int cols) {
 
-        final int weightVectorSizeWithIntercept = data.colSize() + 1;
+        assert datasetBuilder != null;
+
+        PartitionDataBuilder<K, V, SVMPartitionContext, LabeledDataset<Double, LabeledVector>> partDataBuilder = new SVMPartitionDataBuilderOnHeap<>(
+            featureExtractor,
+            lbExtractor,
+            cols
+        );
+
+        this.dataset = datasetBuilder.build(
+            (upstream, upstreamSize) -> new SVMPartitionContext(),
+            partDataBuilder
+        );
+
+
+        final int weightVectorSizeWithIntercept = cols + 1;
         Vector weights = initializeWeightsWithZeros(weightVectorSizeWithIntercept);
 
         for (int i = 0; i < this.getAmountOfIterations(); i++) {
-            Vector deltaWeights = calculateUpdates(data, weights);
+            Vector deltaWeights = calculateUpdates(weights);
             weights = weights.plus(deltaWeights); // creates new vector
         }
 
@@ -67,34 +87,32 @@ public class SVMLinearBinaryClassificationTrainer implements Trainer<SVMLinearBi
 
     /** */
     @NotNull private Vector initializeWeightsWithZeros(int vectorSize) {
-        if (isDistributed)
-            return new SparseDistributedVector(vectorSize);
-        else
             return new DenseLocalOnHeapVector(vectorSize);
     }
 
     /** */
-    private Vector calculateUpdates(LabeledDataset data, Vector weights) {
-        Vector copiedWeights = weights.copy();
-        Vector deltaWeights = initializeWeightsWithZeros(weights.size());
-
-        final int amountOfObservation = data.rowSize();
+    private Vector calculateUpdates(Vector weights) {
+        return dataset.compute(data -> {
+            Vector copiedWeights = weights.copy();
+            Vector deltaWeights = initializeWeightsWithZeros(weights.size());
+            final int amountOfObservation = data.rowSize();
 
-        Vector tmpAlphas = initializeWeightsWithZeros(amountOfObservation);
-        Vector deltaAlphas = initializeWeightsWithZeros(amountOfObservation);
+            Vector tmpAlphas = initializeWeightsWithZeros(amountOfObservation);
+            Vector deltaAlphas = initializeWeightsWithZeros(amountOfObservation);
 
-        for (int i = 0; i < this.getAmountOfLocIterations(); i++) {
-            int randomIdx = ThreadLocalRandom.current().nextInt(amountOfObservation);
+            for (int i = 0; i < this.getAmountOfLocIterations(); i++) {
+                int randomIdx = ThreadLocalRandom.current().nextInt(amountOfObservation);
 
-            Deltas deltas = getDeltas(data, copiedWeights, amountOfObservation, tmpAlphas, randomIdx);
+                Deltas deltas = getDeltas(data, copiedWeights, amountOfObservation, tmpAlphas, randomIdx);
 
-            copiedWeights = copiedWeights.plus(deltas.deltaWeights); // creates new vector
-            deltaWeights = deltaWeights.plus(deltas.deltaWeights);  // creates new vector
+                copiedWeights = copiedWeights.plus(deltas.deltaWeights); // creates new vector
+                deltaWeights = deltaWeights.plus(deltas.deltaWeights);  // creates new vector
 
-            tmpAlphas.set(randomIdx, tmpAlphas.get(randomIdx) + deltas.deltaAlpha);
-            deltaAlphas.set(randomIdx, deltaAlphas.get(randomIdx) + deltas.deltaAlpha);
-        }
-        return deltaWeights;
+                tmpAlphas.set(randomIdx, tmpAlphas.get(randomIdx) + deltas.deltaAlpha);
+                deltaAlphas.set(randomIdx, deltaAlphas.get(randomIdx) + deltas.deltaAlpha);
+            }
+            return deltaWeights;
+        }, (a, b) -> a == null ? b : a.plus(b));
     }
 
     /** */
@@ -225,6 +243,7 @@ public class SVMLinearBinaryClassificationTrainer implements Trainer<SVMLinearBi
         this.amountOfLocIterations = amountOfLocIterations;
         return this;
     }
+
 }
 
 /** This is a helper class to handle pair results which are returned from the calculation method. */

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
deleted file mode 100644
index fd91595..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import java.util.TreeMap;
-import org.apache.ignite.ml.Exportable;
-import org.apache.ignite.ml.Exporter;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.math.Vector;
-
-/** Base class for multi-classification model for set of SVM classifiers. */
-public class SVMLinearMultiClassClassificationModel implements Model<Vector, Double>, Exportable<SVMLinearMultiClassClassificationModel>, Serializable {
-    /** List of models associated with each class. */
-    private Map<Double, SVMLinearBinaryClassificationModel> models;
-
-    /** */
-    public SVMLinearMultiClassClassificationModel() {
-        this.models = new HashMap<>();
-    }
-
-    /** {@inheritDoc} */
-    @Override public Double apply(Vector input) {
-        TreeMap<Double, Double> maxMargins = new TreeMap<>();
-
-        models.forEach((k, v) -> maxMargins.put(input.dot(v.weights()) + v.intercept(), k));
-
-        return maxMargins.lastEntry().getValue();
-    }
-
-    /** {@inheritDoc} */
-    @Override public <P> void saveModel(Exporter<SVMLinearMultiClassClassificationModel, P> exporter, P path) {
-        exporter.save(this, path);
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean equals(Object o) {
-        if (this == o)
-            return true;
-        if (o == null || getClass() != o.getClass())
-            return false;
-        SVMLinearMultiClassClassificationModel mdl = (SVMLinearMultiClassClassificationModel)o;
-        return Objects.equals(models, mdl.models);
-    }
-
-    /** {@inheritDoc} */
-    @Override public int hashCode() {
-        return Objects.hash(models);
-    }
-
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        StringBuilder wholeStr = new StringBuilder();
-
-        models.forEach((clsLb, mdl) -> {
-            wholeStr.append("The class with label " + clsLb + " has classifier: " + mdl.toString() + System.lineSeparator());
-        });
-
-        return wholeStr.toString();
-    }
-
-    /**
-     * Adds a specific SVM binary classifier to the bunch of same classifiers.
-     *
-     * @param clsLb The class label for the added model.
-     * @param mdl The model.
-     */
-    public void add(double clsLb, SVMLinearBinaryClassificationModel mdl) {
-        models.put(clsLb, mdl);
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
deleted file mode 100644
index 669e2e3..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.structures.LabeledDataset;
-
-/**
- * Base class for a soft-margin SVM linear multiclass-classification trainer based on the communication-efficient
- * distributed dual coordinate ascent algorithm (CoCoA) with hinge-loss function.
- *
- * All common parameters are shared with bunch of binary classification trainers.
- */
-public class SVMLinearMultiClassClassificationTrainer implements Trainer<SVMLinearMultiClassClassificationModel, LabeledDataset> {
-    /** Amount of outer SDCA algorithm iterations. */
-    private int amountOfIterations = 20;
-
-    /** Amount of local SDCA algorithm iterations. */
-    private int amountOfLocIterations = 50;
-
-    /** Regularization parameter. */
-    private double lambda = 0.2;
-
-    /**
-     * Returns model based on data.
-     *
-     * @param data data to build model.
-     * @return model.
-     */
-    @Override public SVMLinearMultiClassClassificationModel train(LabeledDataset data) {
-        List<Double> classes = getClassLabels(data);
-
-        SVMLinearMultiClassClassificationModel multiClsMdl = new SVMLinearMultiClassClassificationModel();
-
-        classes.forEach(clsLb -> {
-            LabeledDataset binarizedDataset = binarizeLabels(data, clsLb);
-
-            SVMLinearBinaryClassificationTrainer trainer = new SVMLinearBinaryClassificationTrainer()
-                .withAmountOfIterations(this.amountOfIterations())
-                .withAmountOfLocIterations(this.amountOfLocIterations())
-                .withLambda(this.lambda());
-
-            multiClsMdl.add(clsLb, trainer.train(binarizedDataset));
-        });
-
-        return multiClsMdl;
-    }
-
-    /**
-     * Copies the given data and changes class labels in +1 for chosen class and in -1 for the rest classes.
-     *
-     * @param data Data to transform.
-     * @param clsLb Chosen class in schema One-vs-Rest.
-     * @return Copy of dataset with new labels.
-     */
-    private LabeledDataset binarizeLabels(LabeledDataset data, double clsLb) {
-        final LabeledDataset ds = data.copy();
-
-        for (int i = 0; i < ds.rowSize(); i++)
-            ds.setLabel(i, ds.label(i) == clsLb ? 1.0 : -1.0);
-
-        return ds;
-    }
-
-    /** Iterates among dataset and collects class labels. */
-    private List<Double> getClassLabels(LabeledDataset data) {
-        final Set<Double> clsLabels = new HashSet<>();
-
-        for (int i = 0; i < data.rowSize(); i++)
-            clsLabels.add(data.label(i));
-
-        List<Double> res = new ArrayList<>();
-        res.addAll(clsLabels);
-
-        return res;
-    }
-
-    /**
-     * Set up the regularization parameter.
-     *
-     * @param lambda The regularization parameter. Should be more than 0.0.
-     * @return Trainer with new lambda parameter value.
-     */
-    public SVMLinearMultiClassClassificationTrainer withLambda(double lambda) {
-        assert lambda > 0.0;
-        this.lambda = lambda;
-        return this;
-    }
-
-    /**
-     * Gets the regularization lambda.
-     *
-     * @return The parameter value.
-     */
-    public double lambda() {
-        return lambda;
-    }
-
-    /**
-     * Gets the amount of outer iterations of SCDA algorithm.
-     *
-     * @return The parameter value.
-     */
-    public int amountOfIterations() {
-        return amountOfIterations;
-    }
-
-    /**
-     * Set up the amount of outer iterations of SCDA algorithm.
-     *
-     * @param amountOfIterations The parameter value.
-     * @return Trainer with new amountOfIterations parameter value.
-     */
-    public SVMLinearMultiClassClassificationTrainer withAmountOfIterations(int amountOfIterations) {
-        this.amountOfIterations = amountOfIterations;
-        return this;
-    }
-
-    /**
-     * Gets the amount of local iterations of SCDA algorithm.
-     *
-     * @return The parameter value.
-     */
-    public int amountOfLocIterations() {
-        return amountOfLocIterations;
-    }
-
-    /**
-     * Set up the amount of local iterations of SCDA algorithm.
-     *
-     * @param amountOfLocIterations The parameter value.
-     * @return Trainer with new amountOfLocIterations parameter value.
-     */
-    public SVMLinearMultiClassClassificationTrainer withAmountOfLocIterations(int amountOfLocIterations) {
-        this.amountOfLocIterations = amountOfLocIterations;
-        return this;
-    }
-}
-
-
-

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionContext.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionContext.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionContext.java
new file mode 100644
index 0000000..0aee0fb
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionContext.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.svm;
+
+import java.io.Serializable;
+
+/**
+ * Partition context of the SVM classification algorithm.
+ */
+public class SVMPartitionContext implements Serializable {
+    /** */
+    private static final long serialVersionUID = -7212307112344430126L;
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionDataBuilderOnHeap.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionDataBuilderOnHeap.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionDataBuilderOnHeap.java
new file mode 100644
index 0000000..ad85758
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionDataBuilderOnHeap.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.svm;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import org.apache.ignite.ml.dataset.PartitionDataBuilder;
+import org.apache.ignite.ml.dataset.UpstreamEntry;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.structures.LabeledDataset;
+import org.apache.ignite.ml.structures.LabeledVector;
+
+/**
+ * SVM partition data builder that builds {@link LabeledDataset}.
+ *
+ * @param <K> Type of a key in <tt>upstream</tt> data.
+ * @param <V> Type of a value in <tt>upstream</tt> data.
+ * @param <C> Type of a partition <tt>context</tt>.
+ */
+public class SVMPartitionDataBuilderOnHeap<K, V, C extends Serializable>
+    implements PartitionDataBuilder<K, V, C, LabeledDataset<Double, LabeledVector>> {
+    /** */
+    private static final long serialVersionUID = -7820760153954269227L;
+
+    /** Extractor of X matrix row. */
+    private final IgniteBiFunction<K, V, double[]> xExtractor;
+
+    /** Extractor of Y vector value. */
+    private final IgniteBiFunction<K, V, Double> yExtractor;
+
+    /** Number of columns. */
+    private final int cols;
+
+    /**
+     * Constructs a new instance of SVM partition data builder.
+     *
+     * @param xExtractor Extractor of X matrix row.
+     * @param yExtractor Extractor of Y vector value.
+     * @param cols       Number of columns.
+     */
+    public SVMPartitionDataBuilderOnHeap(IgniteBiFunction<K, V, double[]> xExtractor,
+                                         IgniteBiFunction<K, V, Double> yExtractor, int cols) {
+        this.xExtractor = xExtractor;
+        this.yExtractor = yExtractor;
+        this.cols = cols;
+    }
+
+    /** {@inheritDoc} */
+    @Override public LabeledDataset<Double, LabeledVector> build(Iterator<UpstreamEntry<K, V>> upstreamData, long upstreamDataSize,
+                                                                 C ctx) {
+
+        double[][] x = new double[Math.toIntExact(upstreamDataSize)][cols];
+        double[] y = new double[Math.toIntExact(upstreamDataSize)];
+
+        int ptr = 0;
+        while (upstreamData.hasNext()) {
+            UpstreamEntry<K, V> entry = upstreamData.next();
+            double[] row = xExtractor.apply(entry.getKey(), entry.getValue());
+
+            assert row.length == cols : "X extractor must return exactly " + cols + " columns";
+
+            x[ptr] = row;
+
+            y[ptr] = yExtractor.apply(entry.getKey(), entry.getValue());
+
+            ptr++;
+        }
+
+        return new LabeledDataset<>(x, y);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java
index 57d93d6..421a19f 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java
@@ -32,7 +32,6 @@ import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
 import org.apache.ignite.ml.structures.LabeledDataset;
 import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
-import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationModel;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -98,33 +97,6 @@ public class LocalModelsTest {
 
 
     /** */
-    @Test
-    public void importExportSVMMulticlassClassificationModelTest() throws IOException {
-        executeModelTest(mdlFilePath -> {
-
-
-            SVMLinearBinaryClassificationModel binaryMdl1 = new SVMLinearBinaryClassificationModel(new DenseLocalOnHeapVector(new double[]{1, 2}), 3);
-            SVMLinearBinaryClassificationModel binaryMdl2 = new SVMLinearBinaryClassificationModel(new DenseLocalOnHeapVector(new double[]{2, 3}), 4);
-            SVMLinearBinaryClassificationModel binaryMdl3 = new SVMLinearBinaryClassificationModel(new DenseLocalOnHeapVector(new double[]{3, 4}), 5);
-
-            SVMLinearMultiClassClassificationModel mdl = new SVMLinearMultiClassClassificationModel();
-            mdl.add(1, binaryMdl1);
-            mdl.add(2, binaryMdl2);
-            mdl.add(3, binaryMdl3);
-
-            Exporter<SVMLinearMultiClassClassificationModel, String> exporter = new FileExporter<>();
-            mdl.saveModel(exporter, mdlFilePath);
-
-            SVMLinearMultiClassClassificationModel load = exporter.load(mdlFilePath);
-
-            Assert.assertNotNull(load);
-            Assert.assertEquals("", mdl, load);
-
-            return null;
-        });
-    }
-
-    /** */
     private void executeModelTest(Function<String, Void> code) throws IOException {
         Path mdlPath = Files.createTempFile(null, null);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/BaseSVMTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/BaseSVMTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/BaseSVMTest.java
deleted file mode 100644
index 424118d..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/BaseSVMTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Base class for SVM tests.
- */
-public class BaseSVMTest extends GridCommonAbstractTest {
-    /** Count of nodes. */
-    private static final int NODE_COUNT = 4;
-
-    /** Grid instance. */
-    protected Ignite ignite;
-
-    /**
-     * Default constructor.
-     */
-    public BaseSVMTest() {
-        super(false);
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override protected void beforeTest() throws Exception {
-        ignite = grid(NODE_COUNT);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        for (int i = 1; i <= NODE_COUNT; i++)
-            startGrid(i);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        stopAllGrids();
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
new file mode 100644
index 0000000..353915c
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.svm;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.ml.TestUtils;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.junit.Test;
+
+/**
+ * Tests for {@link SVMLinearBinaryClassificationTrainer}.
+ */
+public class SVMBinaryTrainerTest {
+    /** Fixed size of Dataset. */
+    private static final int AMOUNT_OF_OBSERVATIONS = 1000;
+
+    /** Fixed size of columns in Dataset. */
+    private static final int AMOUNT_OF_FEATURES = 2;
+
+    /**
+     * Test trainer on classification model y = x.
+     */
+    @Test
+    public void testTrainWithTheLinearlySeparableCase() {
+        Map<Integer, double[]> data = new HashMap<>();
+
+
+        ThreadLocalRandom rndX = ThreadLocalRandom.current();
+        ThreadLocalRandom rndY = ThreadLocalRandom.current();
+
+        for (int i = 0; i < AMOUNT_OF_OBSERVATIONS; i++) {
+            double x = rndX.nextDouble(-1000, 1000);
+            double y = rndY.nextDouble(-1000, 1000);
+            double[] vec = new double[AMOUNT_OF_FEATURES + 1];
+            vec[0] = y - x > 0 ? 1 : -1; // assign label.
+            vec[1] = x;
+            vec[2] = y;
+            data.put(i, vec);
+        }
+
+
+        SVMLinearBinaryClassificationTrainer<Integer, double[]> trainer = new SVMLinearBinaryClassificationTrainer<>();
+
+        SVMLinearBinaryClassificationModel mdl = trainer.fit(
+            new LocalDatasetBuilder<>(data, 10),
+            (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+            (k, v) -> v[0],
+            AMOUNT_OF_FEATURES);
+
+        double precision = 1e-2;
+
+        TestUtils.assertEquals(-1, mdl.apply(new DenseLocalOnHeapVector(new double[]{100, 10})), precision);
+        TestUtils.assertEquals(1, mdl.apply(new DenseLocalOnHeapVector(new double[]{10, 100})), precision);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMModelTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMModelTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMModelTest.java
index 35b6644..2533466 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMModelTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMModelTest.java
@@ -59,21 +59,6 @@ public class SVMModelTest {
 
     /** */
     @Test
-    public void testPredictWithMultiClasses() {
-        Vector weights1 = new DenseLocalOnHeapVector(new double[]{10.0, 0.0});
-        Vector weights2 = new DenseLocalOnHeapVector(new double[]{0.0, 10.0});
-        Vector weights3 = new DenseLocalOnHeapVector(new double[]{-1.0, -1.0});
-        SVMLinearMultiClassClassificationModel mdl = new SVMLinearMultiClassClassificationModel();
-        mdl.add(1, new SVMLinearBinaryClassificationModel(weights1, 0.0).withRawLabels(true));
-        mdl.add(2, new SVMLinearBinaryClassificationModel(weights2, 0.0).withRawLabels(true));
-        mdl.add(2, new SVMLinearBinaryClassificationModel(weights3, 0.0).withRawLabels(true));
-
-        Vector observation = new DenseLocalOnHeapVector(new double[]{1.0, 1.0});
-        TestUtils.assertEquals( 1.0, mdl.apply(observation), PRECISION);
-    }
-
-    /** */
-    @Test
     public void testPredictWithErasedLabels() {
         Vector weights = new DenseLocalOnHeapVector(new double[]{1.0, 1.0});
         SVMLinearBinaryClassificationModel mdl = new SVMLinearBinaryClassificationModel(weights, 1.0);

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMTestSuite.java
index 853a43f..dd87fec 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMTestSuite.java
@@ -17,23 +17,16 @@
 
 package org.apache.ignite.ml.svm;
 
-import org.apache.ignite.ml.svm.binary.DistributedLinearSVMBinaryClassificationTrainerTest;
-import org.apache.ignite.ml.svm.binary.LocalLinearSVMBinaryClassificationTrainerTest;
-import org.apache.ignite.ml.svm.multi.DistributedLinearSVMMultiClassClassificationTrainerTest;
-import org.apache.ignite.ml.svm.multi.LocalLinearSVMMultiClassClassificationTrainerTest;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
 
 /**
- * Test suite for all tests located in org.apache.ignite.ml.regressions.* package.
+ * Test suite for all tests located in org.apache.ignite.ml.svm.* package.
  */
 @RunWith(Suite.class)
 @Suite.SuiteClasses({
-    LocalLinearSVMBinaryClassificationTrainerTest.class,
-    DistributedLinearSVMBinaryClassificationTrainerTest.class,
-    LocalLinearSVMMultiClassClassificationTrainerTest.class,
-    DistributedLinearSVMMultiClassClassificationTrainerTest.class,
-    SVMModelTest.class
+    SVMModelTest.class,
+    SVMBinaryTrainerTest.class
 })
 public class SVMTestSuite {
     // No-op.

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/DistributedLinearSVMBinaryClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/DistributedLinearSVMBinaryClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/DistributedLinearSVMBinaryClassificationTrainerTest.java
deleted file mode 100644
index 1be1d1c..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/DistributedLinearSVMBinaryClassificationTrainerTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm.binary;
-
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
-import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationTrainer;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link DenseLocalOnHeapMatrix}.
- */
-public class DistributedLinearSVMBinaryClassificationTrainerTest extends GenericLinearSVMBinaryClassificationTrainerTest {
-    /** */
-    public DistributedLinearSVMBinaryClassificationTrainerTest() {
-        super(
-            new SVMLinearBinaryClassificationTrainer(),
-            true,
-            1e-2);
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/GenericLinearSVMBinaryClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/GenericLinearSVMBinaryClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/GenericLinearSVMBinaryClassificationTrainerTest.java
deleted file mode 100644
index f390557..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/GenericLinearSVMBinaryClassificationTrainerTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm.binary;
-
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledDataset;
-import org.apache.ignite.ml.structures.LabeledVector;
-import org.apache.ignite.ml.svm.BaseSVMTest;
-import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
-import org.junit.Test;
-
-/**
- * Base class for all linear regression trainers.
- */
-public class GenericLinearSVMBinaryClassificationTrainerTest extends BaseSVMTest {
-    /** Fixed size of Dataset. */
-    private static final int AMOUNT_OF_OBSERVATIONS = 100;
-
-    /** Fixed size of columns in Dataset. */
-    private static final int AMOUNT_OF_FEATURES = 2;
-
-    /** */
-    private final Trainer<SVMLinearBinaryClassificationModel, LabeledDataset> trainer;
-
-    /** */
-    private boolean isDistributed;
-
-    /** */
-    private final double precision;
-
-    /** */
-    GenericLinearSVMBinaryClassificationTrainerTest(
-        Trainer<SVMLinearBinaryClassificationModel, LabeledDataset> trainer,
-        boolean isDistributed,
-        double precision) {
-        super();
-        this.trainer = trainer;
-        this.precision = precision;
-        this.isDistributed = isDistributed;
-    }
-
-    /**
-     * Test trainer on classification model y = x.
-     */
-    @Test
-    public void testTrainWithTheLinearlySeparableCase() {
-        if (isDistributed)
-            IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-        LabeledDataset dataset = new LabeledDataset<Double, LabeledVector>(AMOUNT_OF_OBSERVATIONS, AMOUNT_OF_FEATURES, isDistributed);
-
-        ThreadLocalRandom rndX = ThreadLocalRandom.current();
-        ThreadLocalRandom rndY = ThreadLocalRandom.current();
-        for (int i = 0; i < AMOUNT_OF_OBSERVATIONS; i++) {
-            double x = rndX.nextDouble(-1000, 1000);
-            double y = rndY.nextDouble(-1000, 1000);
-            dataset.features(i).set(0, x);
-            dataset.features(i).set(1, y);
-            double lb = y - x > 0 ? 1 : -1;
-            dataset.setLabel(i, lb);
-        }
-
-        SVMLinearBinaryClassificationModel mdl = trainer.train(dataset);
-
-        TestUtils.assertEquals(-1, mdl.apply(new DenseLocalOnHeapVector(new double[] {100, 10})), precision);
-        TestUtils.assertEquals(1, mdl.apply(new DenseLocalOnHeapVector(new double[] {10, 100})), precision);
-    }
-
-    /**
-     * Test trainer on classification model y = x. Amount of generated points is increased 10 times.
-     */
-    @Test
-    public void testTrainWithTheLinearlySeparableCase10() {
-        if (isDistributed)
-            IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-        LabeledDataset dataset = new LabeledDataset<Double, LabeledVector>(AMOUNT_OF_OBSERVATIONS * 10, AMOUNT_OF_FEATURES, isDistributed);
-
-        ThreadLocalRandom rndX = ThreadLocalRandom.current();
-        ThreadLocalRandom rndY = ThreadLocalRandom.current();
-        for (int i = 0; i < AMOUNT_OF_OBSERVATIONS * 10; i++) {
-            double x = rndX.nextDouble(-1000, 1000);
-            double y = rndY.nextDouble(-1000, 1000);
-            dataset.features(i).set(0, x);
-            dataset.features(i).set(1, y);
-            double lb = y - x > 0 ? 1 : -1;
-            dataset.setLabel(i, lb);
-        }
-
-        SVMLinearBinaryClassificationModel mdl = trainer.train(dataset);
-
-        TestUtils.assertEquals(-1, mdl.apply(new DenseLocalOnHeapVector(new double[] {100, 10})), precision);
-        TestUtils.assertEquals(1, mdl.apply(new DenseLocalOnHeapVector(new double[] {10, 100})), precision);
-    }
-
-    /**
-     * Test trainer on classification model y = x. Amount of generated points is increased 100 times.
-     */
-    @Test
-    public void testTrainWithTheLinearlySeparableCase100() {
-        if (isDistributed)
-            IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-        LabeledDataset dataset = new LabeledDataset<Double, LabeledVector>(AMOUNT_OF_OBSERVATIONS * 100, AMOUNT_OF_FEATURES, isDistributed);
-
-        ThreadLocalRandom rndX = ThreadLocalRandom.current();
-        ThreadLocalRandom rndY = ThreadLocalRandom.current();
-        for (int i = 0; i < AMOUNT_OF_OBSERVATIONS * 100; i++) {
-            double x = rndX.nextDouble(-1000, 1000);
-            double y = rndY.nextDouble(-1000, 1000);
-            dataset.features(i).set(0, x);
-            dataset.features(i).set(1, y);
-            double lb = y - x > 0 ? 1 : -1;
-            dataset.setLabel(i, lb);
-        }
-
-        SVMLinearBinaryClassificationModel mdl = trainer.train(dataset);
-
-        TestUtils.assertEquals(-1, mdl.apply(new DenseLocalOnHeapVector(new double[] {100, 10})), precision);
-        TestUtils.assertEquals(1, mdl.apply(new DenseLocalOnHeapVector(new double[] {10, 100})), precision);
-    }
-}


[32/50] [abbrv] ignite git commit: IGNITE-7727: Test unmuted. Fixed by #70ca86a - Fixes #3637.

Posted by sb...@apache.org.
IGNITE-7727: Test unmuted. Fixed by #70ca86a - Fixes #3637.

Signed-off-by: Nikolay Izhikov <ni...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/90e0bf0c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/90e0bf0c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/90e0bf0c

Branch: refs/heads/ignite-zk
Commit: 90e0bf0c1de9b7e372eea52734318f571c5939fa
Parents: 70ca86a
Author: Nikolay Izhikov <ni...@apache.org>
Authored: Wed Mar 14 23:03:26 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Wed Mar 14 23:03:26 2018 +0300

----------------------------------------------------------------------
 .../src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala   | 4 ----
 1 file changed, 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/90e0bf0c/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala
index 2ab3681..cd63e8e 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala
@@ -91,8 +91,6 @@ class IgniteRDDSpec extends FunSpec with Matchers with BeforeAndAfterAll with Be
         }
 
         it("should successfully store data to ignite using saveValues") {
-            fail("https://issues.apache.org/jira/browse/IGNITE-7727")
-
             val sc = new SparkContext("local[*]", "test")
 
             try {
@@ -116,8 +114,6 @@ class IgniteRDDSpec extends FunSpec with Matchers with BeforeAndAfterAll with Be
         }
 
         it("should successfully store data to ignite using saveValues with inline transformation") {
-            fail("https://issues.apache.org/jira/browse/IGNITE-7727")
-
             val sc = new SparkContext("local[*]", "test")
 
             try {


[38/50] [abbrv] ignite git commit: IGNITE-6390 Web-console: Clusters sorted alphabetically.

Posted by sb...@apache.org.
IGNITE-6390 Web-console: Clusters sorted alphabetically.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6d851779
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6d851779
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6d851779

Branch: refs/heads/ignite-zk
Commit: 6d8517796aa1a121d0a6ba5fac7154c05a310eee
Parents: 682197b
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Fri Mar 16 17:05:33 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Mar 16 17:05:33 2018 +0700

----------------------------------------------------------------------
 .../frontend/app/components/cluster-selector/controller.js       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/6d851779/modules/web-console/frontend/app/components/cluster-selector/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/cluster-selector/controller.js b/modules/web-console/frontend/app/components/cluster-selector/controller.js
index 6a86357..d47432b 100644
--- a/modules/web-console/frontend/app/components/cluster-selector/controller.js
+++ b/modules/web-console/frontend/app/components/cluster-selector/controller.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
+
 export default class {
     static $inject = ['$scope', 'AgentManager', 'IgniteConfirm'];
 
@@ -29,7 +31,7 @@ export default class {
         this.clusters$ = this.agentMgr.connectionSbj
             .do(({ cluster, clusters }) => {
                 this.cluster = cluster;
-                this.clusters = clusters;
+                this.clusters = _.orderBy(clusters, ['name'], ['asc']);
             })
             .subscribe(() => {});
     }


[33/50] [abbrv] ignite git commit: .NET: Fix code inspection errors

Posted by sb...@apache.org.
.NET: Fix code inspection errors


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a69e1acb
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a69e1acb
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a69e1acb

Branch: refs/heads/ignite-zk
Commit: a69e1acba8a69f1818fa9182a1167302f81b93ec
Parents: 90e0bf0
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Thu Mar 15 00:05:32 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Thu Mar 15 00:05:32 2018 +0300

----------------------------------------------------------------------
 .../Apache.Ignite.Core/Compute/IComputeTask.cs  |  2 ++
 .../Query/Continuous/ContinuousQueryFilter.cs   |  2 ++
 .../Impl/Common/IgniteArgumentCheck.cs          |  1 +
 .../Impl/Handle/HandleRegistry.cs               |  1 +
 .../Impl/Unmanaged/Jni/Jvm.cs                   |  1 +
 .../Apache.Ignite.Core/Ssl/SslContextFactory.cs |  8 +++++--
 .../Impl/CacheFieldsQueryProvider.cs            | 23 +++++++++++---------
 7 files changed, 26 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTask.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTask.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTask.cs
index 5ae1420..353e087 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTask.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTask.cs
@@ -102,6 +102,8 @@ namespace Apache.Ignite.Core.Compute
         /// <param name="rcvd">All previously received results. Note that if task class has
         /// <see cref="ComputeTaskNoResultCacheAttribute"/> attribute, then this list will be empty.</param>
         /// <returns>Result policy that dictates how to process further upcoming job results.</returns>
+        // ReSharper disable once UnusedParameter.Global
+        // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Global
         ComputeJobResultPolicy OnResult(IComputeJobResult<TJobRes> res, IList<IComputeJobResult<TJobRes>> rcvd);
 
         /// <summary>

http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryFilter.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryFilter.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryFilter.cs
index 7cdb91b..2499463 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryFilter.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryFilter.cs
@@ -83,6 +83,7 @@ namespace Apache.Ignite.Core.Impl.Cache.Query.Continuous
         /** <inheritDoc /> */
         public bool Evaluate(IBinaryStream stream)
         {
+            // ReSharper disable once InconsistentlySynchronizedField
             ICacheEntryEvent<TK, TV> evt = CQU.ReadEvent<TK, TV>(stream, _ignite.Marshaller, _keepBinary);
 
             return _filter.Evaluate(evt);
@@ -91,6 +92,7 @@ namespace Apache.Ignite.Core.Impl.Cache.Query.Continuous
         /** <inheritDoc /> */
         public void Inject(Ignite grid)
         {
+            // ReSharper disable once InconsistentlySynchronizedField
             _ignite = grid;
 
             ResourceProcessor.Inject(_filter, grid);

http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteArgumentCheck.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteArgumentCheck.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteArgumentCheck.cs
index a633291..967225b 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteArgumentCheck.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteArgumentCheck.cs
@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 
+// ReSharper disable ParameterOnlyUsedForPreconditionCheck.Global
 namespace Apache.Ignite.Core.Impl.Common
 {
     using System;

http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Handle/HandleRegistry.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Handle/HandleRegistry.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Handle/HandleRegistry.cs
index fee7d79..d3d4b30 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Handle/HandleRegistry.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Handle/HandleRegistry.cs
@@ -239,6 +239,7 @@ namespace Apache.Ignite.Core.Impl.Handle
         /// <param name="id">Identifier.</param>
         /// <param name="throwOnAbsent">Whether to throw an exception if resource is not found.</param>
         /// <returns>Target.</returns>
+        // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Global
         public T Get<T>(long id, bool throwOnAbsent)
         {
             object target;

http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs
index 6d706e8..5c781b5 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs
@@ -145,6 +145,7 @@ namespace Apache.Ignite.Core.Impl.Unmanaged.Jni
         /// <summary>
         /// Gets the JVM.
         /// </summary>
+        // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Global
         public static Jvm Get(bool ignoreMissing = false)
         {
             var res = _instance;

http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Core/Ssl/SslContextFactory.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Ssl/SslContextFactory.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Ssl/SslContextFactory.cs
index bb5c7a0..a55c6f1 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Ssl/SslContextFactory.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Ssl/SslContextFactory.cs
@@ -48,10 +48,14 @@ namespace Apache.Ignite.Core.Ssl
         /// </summary>
         public string KeyStoreType { get; set; }
 
-        /// <inheritdoc />
+        /// <summary>
+        /// Key store file path.
+        /// </summary>
         public string KeyStoreFilePath { get; set; }
 
-        /// <inheritdoc />
+        /// <summary>
+        /// Key store file password.
+        /// </summary>
         public string KeyStorePassword { get; set; }
 
         /// <summary>

http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryProvider.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryProvider.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryProvider.cs
index 4f35a42..cfc30cc 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryProvider.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryProvider.cs
@@ -220,17 +220,20 @@ namespace Apache.Ignite.Linq.Impl
             // Try with full type name (this works when TableName is not set).
             var valueTypeName = cacheValueType.FullName;
 
-            if (validTableNames.Contains(valueTypeName, StringComparer.OrdinalIgnoreCase))
+            if (valueTypeName != null)
             {
-                return EscapeTableName(valueTypeName);
-            }
-
-            // Remove namespace and nested class qualification and try again.
-            valueTypeName = EscapeTableName(valueTypeName);
-
-            if (validTableNames.Contains(valueTypeName, StringComparer.OrdinalIgnoreCase))
-            {
-                return valueTypeName;
+                if (validTableNames.Contains(valueTypeName, StringComparer.OrdinalIgnoreCase))
+                {
+                    return EscapeTableName(valueTypeName);
+                }
+
+                // Remove namespace and nested class qualification and try again.
+                valueTypeName = EscapeTableName(valueTypeName);
+
+                if (validTableNames.Contains(valueTypeName, StringComparer.OrdinalIgnoreCase))
+                {
+                    return valueTypeName;
+                }
             }
 
             throw new CacheException(string.Format("Table name cannot be inferred for cache '{0}', " +


[39/50] [abbrv] ignite git commit: IGNITE-7929 Web Console: Fixed popover width.

Posted by sb...@apache.org.
IGNITE-7929 Web Console: Fixed popover width.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5b402434
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5b402434
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5b402434

Branch: refs/heads/ignite-zk
Commit: 5b4024342889d396ae19c2224c85c2da20d83e2f
Parents: 6d85177
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Fri Mar 16 17:13:41 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Mar 16 17:13:41 2018 +0700

----------------------------------------------------------------------
 modules/web-console/frontend/public/stylesheets/style.scss | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/5b402434/modules/web-console/frontend/public/stylesheets/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index ece3b65..fe1e94e 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -1183,7 +1183,9 @@ button.form-control {
 }
 
 .theme-line .popover.validation-error {
-    max-width: 400px;
+    white-space: pre-wrap;
+    width: auto !important;
+    max-width: 400px !important;
     color: $brand-primary;
     background: white;
     border: 1px solid $brand-primary;


[30/50] [abbrv] ignite git commit: IGNITE-7898 Fixed IgniteCachePartitionLossPolicy tests - Fixes #3626.

Posted by sb...@apache.org.
IGNITE-7898 Fixed IgniteCachePartitionLossPolicy tests - Fixes #3626.

Signed-off-by: Alexey Goncharuk <al...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e7ee88aa
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e7ee88aa
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e7ee88aa

Branch: refs/heads/ignite-zk
Commit: e7ee88aa82b3c59eb1ae671e2d33f1307f17162e
Parents: 798c01b
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Wed Mar 14 16:51:54 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Mar 14 16:51:54 2018 +0300

----------------------------------------------------------------------
 .../IgniteCachePartitionLossPolicySelfTest.java | 29 ++++++++++++++++----
 1 file changed, 24 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/e7ee88aa/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
index 23d3941..7f35ddb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
@@ -17,11 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache.distributed;
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import javax.cache.CacheException;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.PartitionLossPolicy;
@@ -34,14 +29,22 @@ import org.apache.ignite.events.CacheRebalancingEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.P1;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import javax.cache.CacheException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
 
@@ -158,6 +161,22 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe
 
         int part = prepareTopology();
 
+        // Wait for all grids (servers and client) have same topology version
+        // to make sure that all nodes received map with lost partition.
+        GridTestUtils.waitForCondition(() -> {
+            AffinityTopologyVersion last = null;
+            for (Ignite ig : G.allGrids()) {
+                AffinityTopologyVersion ver = ((IgniteEx) ig).context().cache().context().exchange().readyAffinityVersion();
+
+                if (last != null && !last.equals(ver))
+                    return false;
+
+                last = ver;
+            }
+
+            return true;
+        }, 10000);
+
         for (Ignite ig : G.allGrids()) {
             info("Checking node: " + ig.cluster().localNode().id());
 


[08/50] [abbrv] ignite git commit: IGNITE-7897: Add example for LSQR with data normalization.

Posted by sb...@apache.org.
IGNITE-7897: Add example for LSQR with data normalization.

This closes #3614


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6dbcf7e3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6dbcf7e3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6dbcf7e3

Branch: refs/heads/ignite-zk
Commit: 6dbcf7e34ec3b1d3cd80fc83ab53a9959c3b1fe1
Parents: a8170f7
Author: Anton Dmitriev <dm...@gmail.com>
Authored: Wed Mar 7 15:11:53 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Wed Mar 7 15:11:53 2018 +0300

----------------------------------------------------------------------
 ...nWithLSQRTrainerAndNormalizationExample.java | 184 +++++++++++++++++++
 1 file changed, 184 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/6dbcf7e3/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
new file mode 100644
index 0000000..61195c4
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.regression.linear;
+
+import java.util.Arrays;
+import java.util.UUID;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.examples.ml.math.matrix.SparseDistributedMatrixExample;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.preprocessing.normalization.NormalizationPreprocessor;
+import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
+import org.apache.ignite.ml.regressions.linear.LinearRegressionLSQRTrainer;
+import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * Run linear regression model over distributed matrix.
+ *
+ * @see LinearRegressionLSQRTrainer
+ * @see NormalizationTrainer
+ * @see NormalizationPreprocessor
+ */
+public class DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample {
+    /** */
+    private static final double[][] data = {
+        {8, 78, 284, 9.100000381, 109},
+        {9.300000191, 68, 433, 8.699999809, 144},
+        {7.5, 70, 739, 7.199999809, 113},
+        {8.899999619, 96, 1792, 8.899999619, 97},
+        {10.19999981, 74, 477, 8.300000191, 206},
+        {8.300000191, 111, 362, 10.89999962, 124},
+        {8.800000191, 77, 671, 10, 152},
+        {8.800000191, 168, 636, 9.100000381, 162},
+        {10.69999981, 82, 329, 8.699999809, 150},
+        {11.69999981, 89, 634, 7.599999905, 134},
+        {8.5, 149, 631, 10.80000019, 292},
+        {8.300000191, 60, 257, 9.5, 108},
+        {8.199999809, 96, 284, 8.800000191, 111},
+        {7.900000095, 83, 603, 9.5, 182},
+        {10.30000019, 130, 686, 8.699999809, 129},
+        {7.400000095, 145, 345, 11.19999981, 158},
+        {9.600000381, 112, 1357, 9.699999809, 186},
+        {9.300000191, 131, 544, 9.600000381, 177},
+        {10.60000038, 80, 205, 9.100000381, 127},
+        {9.699999809, 130, 1264, 9.199999809, 179},
+        {11.60000038, 140, 688, 8.300000191, 80},
+        {8.100000381, 154, 354, 8.399999619, 103},
+        {9.800000191, 118, 1632, 9.399999619, 101},
+        {7.400000095, 94, 348, 9.800000191, 117},
+        {9.399999619, 119, 370, 10.39999962, 88},
+        {11.19999981, 153, 648, 9.899999619, 78},
+        {9.100000381, 116, 366, 9.199999809, 102},
+        {10.5, 97, 540, 10.30000019, 95},
+        {11.89999962, 176, 680, 8.899999619, 80},
+        {8.399999619, 75, 345, 9.600000381, 92},
+        {5, 134, 525, 10.30000019, 126},
+        {9.800000191, 161, 870, 10.39999962, 108},
+        {9.800000191, 111, 669, 9.699999809, 77},
+        {10.80000019, 114, 452, 9.600000381, 60},
+        {10.10000038, 142, 430, 10.69999981, 71},
+        {10.89999962, 238, 822, 10.30000019, 86},
+        {9.199999809, 78, 190, 10.69999981, 93},
+        {8.300000191, 196, 867, 9.600000381, 106},
+        {7.300000191, 125, 969, 10.5, 162},
+        {9.399999619, 82, 499, 7.699999809, 95},
+        {9.399999619, 125, 925, 10.19999981, 91},
+        {9.800000191, 129, 353, 9.899999619, 52},
+        {3.599999905, 84, 288, 8.399999619, 110},
+        {8.399999619, 183, 718, 10.39999962, 69},
+        {10.80000019, 119, 540, 9.199999809, 57},
+        {10.10000038, 180, 668, 13, 106},
+        {9, 82, 347, 8.800000191, 40},
+        {10, 71, 345, 9.199999809, 50},
+        {11.30000019, 118, 463, 7.800000191, 35},
+        {11.30000019, 121, 728, 8.199999809, 86},
+        {12.80000019, 68, 383, 7.400000095, 57},
+        {10, 112, 316, 10.39999962, 57},
+        {6.699999809, 109, 388, 8.899999619, 94}
+    };
+
+    /** Run example. */
+    public static void main(String[] args) throws InterruptedException {
+        System.out.println();
+        System.out.println(">>> Linear regression model over sparse distributed matrix API usage example started.");
+        // Start ignite grid.
+        try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+            System.out.println(">>> Ignite grid started.");
+
+            // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
+            // because we create ignite cache internally.
+            IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+                SparseDistributedMatrixExample.class.getSimpleName(), () -> {
+                IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
+
+                System.out.println(">>> Create new normalization trainer object.");
+                NormalizationTrainer<Integer, double[]> normalizationTrainer = new NormalizationTrainer<>();
+
+                System.out.println(">>> Perform the training to get the normalization preprocessor.");
+                NormalizationPreprocessor<Integer, double[]> preprocessor = normalizationTrainer.fit(
+                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
+                    (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+                    4
+                );
+
+                System.out.println(">>> Create new linear regression trainer object.");
+                LinearRegressionLSQRTrainer<Integer, double[]> trainer = new LinearRegressionLSQRTrainer<>();
+
+                System.out.println(">>> Perform the training to get the model.");
+                LinearRegressionModel mdl = trainer.fit(
+                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
+                    preprocessor,
+                    (k, v) -> v[0],
+                    4
+                );
+
+                System.out.println(">>> Linear regression model: " + mdl);
+
+                System.out.println(">>> ---------------------------------");
+                System.out.println(">>> | Prediction\t| Ground Truth\t|");
+                System.out.println(">>> ---------------------------------");
+
+                try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+                    for (Cache.Entry<Integer, double[]> observation : observations) {
+                        Integer key = observation.getKey();
+                        double[] val = observation.getValue();
+                        double groundTruth = val[0];
+
+                        double prediction = mdl.apply(new DenseLocalOnHeapVector(preprocessor.apply(key, val)));
+
+                        System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
+                    }
+                }
+
+                System.out.println(">>> ---------------------------------");
+            });
+
+            igniteThread.start();
+
+            igniteThread.join();
+        }
+    }
+
+    /**
+     * Fills cache with data and returns it.
+     *
+     * @param ignite Ignite instance.
+     * @return Filled Ignite Cache.
+     */
+    private static IgniteCache<Integer, double[]> getTestCache(Ignite ignite) {
+        CacheConfiguration<Integer, double[]> cacheConfiguration = new CacheConfiguration<>();
+        cacheConfiguration.setName("TEST_" + UUID.randomUUID());
+        cacheConfiguration.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+        IgniteCache<Integer, double[]> cache = ignite.createCache(cacheConfiguration);
+
+        for (int i = 0; i < data.length; i++)
+            cache.put(i, data[i]);
+
+        return cache;
+    }
+}


[40/50] [abbrv] ignite git commit: IGNITE-7970 Web Console: Fixed typo in notification test.

Posted by sb...@apache.org.
IGNITE-7970 Web Console: Fixed typo in notification test.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/094f6c32
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/094f6c32
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/094f6c32

Branch: refs/heads/ignite-zk
Commit: 094f6c328b8297cb55335dea6bec391308bbed27
Parents: 5b40243
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Fri Mar 16 17:25:59 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Mar 16 17:25:59 2018 +0700

----------------------------------------------------------------------
 modules/web-console/e2e/testcafe/fixtures/admin-panel.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/094f6c32/modules/web-console/e2e/testcafe/fixtures/admin-panel.js
----------------------------------------------------------------------
diff --git a/modules/web-console/e2e/testcafe/fixtures/admin-panel.js b/modules/web-console/e2e/testcafe/fixtures/admin-panel.js
index 9baf055..5400d79 100644
--- a/modules/web-console/e2e/testcafe/fixtures/admin-panel.js
+++ b/modules/web-console/e2e/testcafe/fixtures/admin-panel.js
@@ -36,7 +36,7 @@ fixture('Checking admin panel')
     });
 
 test('Testing setting notifications', async(t) => {
-    await t.click(Selector('button').withAttribute('ng-click', 'ctrl.changeUserNotifications()'));
+    await t.click(Selector('button').withAttribute('ng-click', '$ctrl.changeUserNotifications()'));
 
     await t
         .expect(Selector('h4').withText(/.*Set user notifications.*/).exists)
@@ -50,7 +50,7 @@ test('Testing setting notifications', async(t) => {
         .expect(Selector('.wch-notification').innerText)
         .eql('test message');
 
-    await t.click(Selector('button').withAttribute('ng-click', 'ctrl.changeUserNotifications()'));
+    await t.click(Selector('button').withAttribute('ng-click', '$ctrl.changeUserNotifications()'));
 
     await t
         .click('.ace_content')


[31/50] [abbrv] ignite git commit: IGNITE-7756: IgniteUuid added to predefined types - Fixes #3550.

Posted by sb...@apache.org.
IGNITE-7756: IgniteUuid added to predefined types - Fixes #3550.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/70ca86a3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/70ca86a3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/70ca86a3

Branch: refs/heads/ignite-zk
Commit: 70ca86a30a7589f9ff466b93a958362135347d02
Parents: e7ee88a
Author: Nikolay Izhikov <ni...@apache.org>
Authored: Wed Mar 14 22:33:47 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Wed Mar 14 22:34:32 2018 +0300

----------------------------------------------------------------------
 .../IgniteUuidCompatibilityTest.java            | 189 +++++++++++++++++++
 .../IgniteCompatibilityBasicTestSuite.java      |   3 +
 .../ignite/internal/binary/BinaryContext.java   |   1 +
 .../client/IgniteDataStreamerTest.java          | 118 ++++++++++++
 .../IgniteCacheWithIndexingTestSuite.java       |   3 +
 5 files changed, 314 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/70ca86a3/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgniteUuidCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgniteUuidCompatibilityTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgniteUuidCompatibilityTest.java
new file mode 100644
index 0000000..88c6900
--- /dev/null
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgniteUuidCompatibilityTest.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.compatibility.persistence;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.binary.BinaryContext;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+
+/**
+ * Checks that index and caches with IgniteUuid are compatible when IgniteUuid becomes predefined type.
+ *
+ * @see BinaryContext#registerPredefinedType(Class, int)
+ */
+public class IgniteUuidCompatibilityTest extends IgnitePersistenceCompatibilityAbstractTest {
+    /** */
+    protected static final String TEST_CACHE_NAME = IgniteUuidCompatibilityTest.class.getSimpleName();
+
+    /** */
+    protected volatile boolean compactFooter;
+
+    /** */
+    private static Set<String> VALUES = new HashSet<>(Arrays.asList("one", "two", "three"));
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "binary_meta", false));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setPeerClassLoadingEnabled(false);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                ));
+
+        cfg.setBinaryConfiguration(
+            new BinaryConfiguration()
+                .setCompactFooter(compactFooter)
+        );
+
+        return cfg;
+    }
+
+    /**
+     * Tests opportunity to read data from previous Ignite DB version for cache contains IgniteUuid instances.
+     *
+     * @throws Exception If failed.
+     */
+    public void testIgniteUuid_2_3() throws Exception {
+        doTestIgniteUuidCompatibility("2.3.0", false);
+    }
+
+    /**
+     * Tests opportunity to read data from previous Ignite DB version for cache contains IgniteUuid instances.
+     *
+     * @throws Exception If failed.
+     */
+    public void testIgniteUuidWithCompactFooter_2_3() throws Exception {
+        doTestIgniteUuidCompatibility("2.3.0", true);
+    }
+
+    /**
+     * Tests opportunity to read data from previous Ignite DB version.
+     *
+     * @param igniteVer 3-digits version of ignite
+     * @throws Exception If failed.
+     */
+    private void doTestIgniteUuidCompatibility(String igniteVer, boolean compactFooter) throws Exception {
+        boolean prev = this.compactFooter;
+
+        try {
+            this.compactFooter = compactFooter;
+
+            startGrid(1, igniteVer, new ConfigurationClosure(compactFooter), new PutData());
+
+            stopAllGrids();
+
+            IgniteEx ignite = startGrid(0);
+
+            assertEquals(1, ignite.context().discovery().topologyVersion());
+
+            ignite.active(true);
+
+            IgniteCache<IgniteUuid, String> cache = ignite.cache(TEST_CACHE_NAME);
+
+            assertEquals(cache.size(CachePeekMode.ALL), VALUES.size());
+
+            Set<String> values = new HashSet<>();
+
+            for (Cache.Entry<IgniteUuid, String> e : cache)
+                values.add(e.getValue());
+
+            assertEquals(values, VALUES);
+        }
+        finally {
+            stopAllGrids();
+
+            this.compactFooter = prev;
+        }
+    }
+
+    /** */
+    public static class ConfigurationClosure implements IgniteInClosure<IgniteConfiguration> {
+        private boolean compactFooter;
+
+        public ConfigurationClosure(boolean compactFooter) {
+            this.compactFooter = compactFooter;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void apply(IgniteConfiguration cfg) {
+            cfg.setLocalHost("127.0.0.1");
+
+            TcpDiscoverySpi disco = new TcpDiscoverySpi();
+            disco.setIpFinder(GridCacheAbstractFullApiSelfTest.LOCAL_IP_FINDER);
+
+            cfg.setDiscoverySpi(disco);
+
+            cfg.setPeerClassLoadingEnabled(false);
+
+            cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
+
+            if (!compactFooter)
+                cfg.setBinaryConfiguration(new BinaryConfiguration().setCompactFooter(compactFooter));
+        }
+    }
+
+    /** */
+    public static class PutData implements IgniteInClosure<Ignite> {
+        /** {@inheritDoc} */
+        @Override public void apply(Ignite ignite) {
+            ignite.active(true);
+
+            CacheConfiguration<IgniteUuid, String> cacheCfg = new CacheConfiguration<>();
+
+            cacheCfg.setName(TEST_CACHE_NAME);
+            cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+            cacheCfg.setBackups(1);
+            cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+
+            IgniteCache<IgniteUuid, String> cache = ignite.createCache(cacheCfg);
+
+            for (String v : VALUES)
+                cache.put(IgniteUuid.randomUuid(), v);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/70ca86a3/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
index 20643d4..eaa38af 100644
--- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
@@ -20,6 +20,7 @@ package org.apache.ignite.compatibility.testsuites;
 import junit.framework.TestSuite;
 import org.apache.ignite.compatibility.persistence.DummyPersistenceCompatibilityTest;
 import org.apache.ignite.compatibility.persistence.FoldersReuseCompatibilityTest;
+import org.apache.ignite.compatibility.persistence.IgniteUuidCompatibilityTest;
 import org.apache.ignite.compatibility.persistence.MigratingToWalV2SerializerWithCompactionTest;
 
 /**
@@ -39,6 +40,8 @@ public class IgniteCompatibilityBasicTestSuite {
 
         suite.addTestSuite(MigratingToWalV2SerializerWithCompactionTest.class);
 
+        suite.addTestSuite(IgniteUuidCompatibilityTest.class);
+
         return suite;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/70ca86a3/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index 9606139..8f85947 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -333,6 +333,7 @@ public class BinaryContext {
         registerPredefinedType(GridMapEntry.class, 60);
         registerPredefinedType(IgniteBiTuple.class, 61);
         registerPredefinedType(T2.class, 62);
+        registerPredefinedType(IgniteUuid.class, 63);
 
         registerPredefinedType(PlatformJavaObjectFactoryProxy.class,
             GridBinaryMarshaller.PLATFORM_JAVA_OBJECT_FACTORY_PROXY);

http://git-wip-us.apache.org/repos/asf/ignite/blob/70ca86a3/modules/indexing/src/test/java/org/apache/ignite/internal/processors/client/IgniteDataStreamerTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/client/IgniteDataStreamerTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/client/IgniteDataStreamerTest.java
new file mode 100644
index 0000000..1bfb02e
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/client/IgniteDataStreamerTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.client;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CachePeekMode.ALL;
+
+/**
+ */
+public class IgniteDataStreamerTest extends GridCommonAbstractTest {
+    public static final String CACHE_NAME = "UUID_CACHE";
+
+    public static final int DATA_SIZE = 3;
+
+    public static final long WAIT_TIMEOUT = 30_000L;
+
+    private boolean client = false;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        if (client)
+            cfg.setClientMode(true);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrids(2);
+
+        client = true;
+
+        startGrid("client");
+    }
+
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        grid("client").destroyCache(CACHE_NAME);
+    }
+
+    /**
+     * @return Cache configuration
+     */
+    private <K, V> CacheConfiguration<K, V> cacheConfiguration(Class<K> key, Class<V> value) {
+        CacheConfiguration<K, V> ccfg = new CacheConfiguration<>(CACHE_NAME);
+
+        ccfg.setIndexedTypes(key, value);
+
+        return ccfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStreamerIgniteUuid() throws Exception {
+        Ignite client = grid("client");
+
+        IgniteCache<IgniteUuid, Integer> cache =
+            client.createCache(cacheConfiguration(IgniteUuid.class, Integer.class));
+
+        try(IgniteDataStreamer<IgniteUuid, Integer> streamer = client.dataStreamer(CACHE_NAME)) {
+            assertTrue("Expecting " + DataStreamerImpl.class.getName(), streamer instanceof DataStreamerImpl);
+
+            ((DataStreamerImpl<IgniteUuid, Integer>)streamer).maxRemapCount(0);
+
+            List<IgniteFuture> futs = new ArrayList<>();
+
+            for(int i=0; i<DATA_SIZE; i++) {
+                IgniteFuture<?> fut = streamer.addData(IgniteUuid.randomUuid(), i);
+
+                futs.add(fut);
+            }
+
+            streamer.flush();
+
+            for (IgniteFuture fut : futs) {
+                //This should not throw any exception.
+                Object res = fut.get(WAIT_TIMEOUT);
+
+                //Printing future result to log to prevent jvm optimization
+                log.debug(res == null ? null : res.toString());
+            }
+
+            assertTrue(cache.size(ALL) == DATA_SIZE);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/70ca86a3/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
index 4047700..bc99981 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
@@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.ttl.CacheTtlAtomicLocalSelfTe
 import org.apache.ignite.internal.processors.cache.ttl.CacheTtlAtomicPartitionedSelfTest;
 import org.apache.ignite.internal.processors.cache.ttl.CacheTtlTransactionalLocalSelfTest;
 import org.apache.ignite.internal.processors.cache.ttl.CacheTtlTransactionalPartitionedSelfTest;
+import org.apache.ignite.internal.processors.client.IgniteDataStreamerTest;
 import org.apache.ignite.internal.processors.query.h2.database.InlineIndexHelperTest;
 
 /**
@@ -78,6 +79,8 @@ public class IgniteCacheWithIndexingTestSuite extends TestSuite {
 
         suite.addTestSuite(IgniteCacheGroupsSqlTest.class);
 
+        suite.addTestSuite(IgniteDataStreamerTest.class);
+
         return suite;
     }
 }


[28/50] [abbrv] ignite git commit: IGNITE-7946 Fail test with known issue - Fixes #3631.

Posted by sb...@apache.org.
IGNITE-7946 Fail test with known issue - Fixes #3631.

Signed-off-by: Alexey Goncharuk <al...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/48a66237
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/48a66237
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/48a66237

Branch: refs/heads/ignite-zk
Commit: 48a66237c1fb2b5e095eed0c0961764b897230f8
Parents: c533e2c
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Wed Mar 14 13:58:20 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Mar 14 13:58:20 2018 +0300

----------------------------------------------------------------------
 .../near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java   | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/48a66237/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
index 996d6a7..de7ee5f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
@@ -209,6 +209,7 @@ public class IgniteCacheClientQueryReplicatedNodeRestartSelfTest extends GridCom
      * @throws Exception If failed.
      */
     public void testRestarts() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-7946");
         int duration = 90 * 1000;
         int qryThreadNum = 5;
         int restartThreadsNum = 2; // 2 of 4 data nodes


[26/50] [abbrv] ignite git commit: IGNITE-7253: JDBC thin driver: implemented streaming. This closes #3499. This closes #3591.

Posted by sb...@apache.org.
IGNITE-7253: JDBC thin driver: implemented streaming. This closes #3499. This closes #3591.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7366809e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7366809e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7366809e

Branch: refs/heads/ignite-zk
Commit: 7366809ed0ea7a943e1c80079fdbeb556ee86245
Parents: ae2bf3d
Author: Alexander Paschenko <al...@gmail.com>
Authored: Wed Mar 14 12:23:37 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Mar 14 12:23:37 2018 +0300

----------------------------------------------------------------------
 .../jdbc2/JdbcNoCacheStreamingSelfTest.java     | 182 +++++++
 .../internal/jdbc2/JdbcStreamingSelfTest.java   | 220 +++++++--
 .../jdbc/suite/IgniteJdbcDriverTestSuite.java   |  11 +-
 .../jdbc/thin/JdbcThinStreamingSelfTest.java    | 486 +++++++++++++++++++
 .../internal/jdbc/thin/JdbcThinConnection.java  | 112 +++++
 .../jdbc/thin/JdbcThinPreparedStatement.java    |  22 +-
 .../internal/jdbc/thin/JdbcThinStatement.java   | 153 +++++-
 .../internal/jdbc/thin/JdbcThinTcpIo.java       |  12 +-
 .../ignite/internal/jdbc2/JdbcConnection.java   |  10 +-
 .../jdbc2/JdbcStreamedPreparedStatement.java    |   2 +-
 .../processors/cache/IgniteCacheProxyImpl.java  |   4 +-
 .../odbc/jdbc/JdbcBatchExecuteRequest.java      |  40 +-
 .../odbc/jdbc/JdbcConnectionContext.java        |  13 +-
 .../odbc/jdbc/JdbcRequestHandler.java           |  84 ++--
 .../processors/query/GridQueryIndexing.java     |  32 +-
 .../processors/query/GridQueryProcessor.java    |  50 +-
 .../processors/query/SqlClientContext.java      | 223 +++++++++
 .../apache/ignite/internal/sql/SqlKeyword.java  |  29 +-
 .../apache/ignite/internal/sql/SqlParser.java   |  26 +-
 .../ignite/internal/sql/SqlParserUtils.java     |  22 +
 .../sql/command/SqlSetStreamingCommand.java     | 191 ++++++++
 ...IgniteClientCacheInitializationFailTest.java |  15 +-
 .../sql/SqlParserSetStreamingSelfTest.java      | 134 +++++
 .../ignite/testframework/GridTestUtils.java     |   2 +-
 .../query/h2/DmlStatementsProcessor.java        | 104 ++--
 .../internal/processors/query/h2/H2Utils.java   |  28 +-
 .../processors/query/h2/IgniteH2Indexing.java   | 134 +++--
 .../query/h2/ddl/DdlStatementsProcessor.java    |  10 +-
 .../processors/query/h2/dml/UpdatePlan.java     |   2 +-
 .../query/h2/dml/UpdatePlanBuilder.java         |  13 +-
 .../query/h2/sql/GridSqlQueryParser.java        |  12 +
 .../cache/GridCacheCrossCacheQuerySelfTest.java |  18 +-
 .../IgniteCacheQuerySelfTestSuite.java          |   2 +
 33 files changed, 2140 insertions(+), 258 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcNoCacheStreamingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcNoCacheStreamingSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcNoCacheStreamingSelfTest.java
new file mode 100644
index 0000000..74c2820
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcNoCacheStreamingSelfTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.jdbc2;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.util.Collections;
+import java.util.Properties;
+import org.apache.ignite.IgniteJdbcDriver;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteJdbcDriver.CFG_URL_PREFIX;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ * Data streaming test for thick driver and no explicit caches.
+ */
+public class JdbcNoCacheStreamingSelfTest extends GridCommonAbstractTest {
+    /** JDBC URL. */
+    private static final String BASE_URL = CFG_URL_PREFIX +
+        "cache=default@modules/clients/src/test/config/jdbc-config.xml";
+
+    /** Connection. */
+    protected Connection conn;
+
+    /** */
+    protected transient IgniteLogger log;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        return getConfiguration0(gridName);
+    }
+
+    /**
+     * @param gridName Grid name.
+     * @return Grid configuration used for starting the grid.
+     * @throws Exception If failed.
+     */
+    private IgniteConfiguration getConfiguration0(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration<?,?> cache = defaultCacheConfiguration();
+
+        cache.setCacheMode(PARTITIONED);
+        cache.setBackups(1);
+        cache.setWriteSynchronizationMode(FULL_SYNC);
+        cache.setIndexedTypes(
+            Integer.class, Integer.class
+        );
+
+        cfg.setCacheConfiguration(cache);
+        cfg.setLocalHost("127.0.0.1");
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+        ipFinder.setAddresses(Collections.singleton("127.0.0.1:47500..47501"));
+
+        disco.setIpFinder(ipFinder);
+
+        cfg.setDiscoverySpi(disco);
+
+        cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrids(2);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @param allowOverwrite Allow overwriting of existing keys.
+     * @return Connection to use for the test.
+     * @throws Exception if failed.
+     */
+    protected Connection createConnection(boolean allowOverwrite) throws Exception {
+        Properties props = new Properties();
+
+        props.setProperty(IgniteJdbcDriver.PROP_STREAMING, "true");
+        props.setProperty(IgniteJdbcDriver.PROP_STREAMING_FLUSH_FREQ, "500");
+
+        if (allowOverwrite)
+            props.setProperty(IgniteJdbcDriver.PROP_STREAMING_ALLOW_OVERWRITE, "true");
+
+        return DriverManager.getConnection(BASE_URL, props);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        U.closeQuiet(conn);
+
+        ignite(0).cache(DEFAULT_CACHE_NAME).clear();
+
+        super.afterTest();
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testStreamedInsert() throws Exception {
+        for (int i = 10; i <= 100; i += 10)
+            ignite(0).cache(DEFAULT_CACHE_NAME).put(i, i * 100);
+
+        try (Connection conn = createConnection(false)) {
+            try (PreparedStatement stmt = conn.prepareStatement("insert into Integer(_key, _val) values (?, ?)")) {
+                for (int i = 1; i <= 100; i++) {
+                    stmt.setInt(1, i);
+                    stmt.setInt(2, i);
+
+                    stmt.executeUpdate();
+                }
+            }
+        }
+
+        U.sleep(500);
+
+        // Now let's check it's all there.
+        for (int i = 1; i <= 100; i++) {
+            if (i % 10 != 0)
+                assertEquals(i, grid(0).cache(DEFAULT_CACHE_NAME).get(i));
+            else // All that divides by 10 evenly should point to numbers 100 times greater - see above
+                assertEquals(i * 100, grid(0).cache(DEFAULT_CACHE_NAME).get(i));
+        }
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testStreamedInsertWithOverwritesAllowed() throws Exception {
+        for (int i = 10; i <= 100; i += 10)
+            ignite(0).cache(DEFAULT_CACHE_NAME).put(i, i * 100);
+
+        try (Connection conn = createConnection(true)) {
+            try (PreparedStatement stmt = conn.prepareStatement("insert into Integer(_key, _val) values (?, ?)")) {
+                for (int i = 1; i <= 100; i++) {
+                    stmt.setInt(1, i);
+                    stmt.setInt(2, i);
+
+                    stmt.executeUpdate();
+                }
+            }
+        }
+
+        U.sleep(500);
+
+        // Now let's check it's all there.
+        // i should point to i at all times as we've turned overwrites on above.
+        for (int i = 1; i <= 100; i++)
+            assertEquals(i, grid(0).cache(DEFAULT_CACHE_NAME).get(i));
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
index 5418ca0..10adedc 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
@@ -20,16 +20,24 @@ package org.apache.ignite.internal.jdbc2;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
 import java.util.Collections;
 import java.util.Properties;
+import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteJdbcDriver;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectBuilder;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.ConnectorConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.query.QueryUtils;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.IgniteJdbcDriver.CFG_URL_PREFIX;
@@ -41,10 +49,12 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
  */
 public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
     /** JDBC URL. */
-    private static final String BASE_URL = CFG_URL_PREFIX + "cache=default@modules/clients/src/test/config/jdbc-config.xml";
+    private static final String BASE_URL = CFG_URL_PREFIX +
+        "cache=default@modules/clients/src/test/config/jdbc-config.xml";
 
-    /** Connection. */
-    protected Connection conn;
+    /** Streaming URL. */
+    private static final String STREAMING_URL = CFG_URL_PREFIX +
+        "cache=person@modules/clients/src/test/config/jdbc-config.xml";
 
     /** */
     protected transient IgniteLogger log;
@@ -90,7 +100,18 @@ public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
 
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
         startGrids(2);
+
+        try (Connection c = createOrdinaryConnection()) {
+            try (Statement s = c.createStatement()) {
+                s.execute("CREATE TABLE PUBLIC.Person(\"id\" int primary key, \"name\" varchar) WITH " +
+                    "\"cache_name=person,value_type=Person\"");
+            }
+        }
+
+        U.sleep(1000);
     }
 
     /** {@inheritDoc} */
@@ -99,27 +120,51 @@ public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
     }
 
     /**
+     * @return Connection without streaming initially turned on.
+     * @throws SQLException if failed.
+     */
+    protected Connection createOrdinaryConnection() throws SQLException {
+        Connection res = DriverManager.getConnection(BASE_URL, new Properties());
+
+        res.setSchema(QueryUtils.DFLT_SCHEMA);
+
+        return res;
+    }
+
+    /**
      * @param allowOverwrite Allow overwriting of existing keys.
      * @return Connection to use for the test.
      * @throws Exception if failed.
      */
-    private Connection createConnection(boolean allowOverwrite) throws Exception {
+    protected Connection createStreamedConnection(boolean allowOverwrite) throws Exception {
+        return createStreamedConnection(allowOverwrite, 500);
+    }
+
+    /**
+     * @param allowOverwrite Allow overwriting of existing keys.
+     * @param flushTimeout Stream flush timeout.
+     * @return Connection to use for the test.
+     * @throws Exception if failed.
+     */
+    protected Connection createStreamedConnection(boolean allowOverwrite, long flushTimeout) throws Exception {
         Properties props = new Properties();
 
         props.setProperty(IgniteJdbcDriver.PROP_STREAMING, "true");
-        props.setProperty(IgniteJdbcDriver.PROP_STREAMING_FLUSH_FREQ, "500");
+        props.setProperty(IgniteJdbcDriver.PROP_STREAMING_FLUSH_FREQ, String.valueOf(flushTimeout));
 
         if (allowOverwrite)
             props.setProperty(IgniteJdbcDriver.PROP_STREAMING_ALLOW_OVERWRITE, "true");
 
-        return DriverManager.getConnection(BASE_URL, props);
+        Connection res = DriverManager.getConnection(STREAMING_URL, props);
+
+        res.setSchema(QueryUtils.DFLT_SCHEMA);
+
+        return res;
     }
 
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
-        U.closeQuiet(conn);
-
-        ignite(0).cache(DEFAULT_CACHE_NAME).clear();
+        cache().clear();
 
         super.afterTest();
     }
@@ -128,30 +173,59 @@ public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
      * @throws Exception if failed.
      */
     public void testStreamedInsert() throws Exception {
-        conn = createConnection(false);
-
         for (int i = 10; i <= 100; i += 10)
-            ignite(0).cache(DEFAULT_CACHE_NAME).put(i, i * 100);
+            put(i, nameForId(i * 100));
+
+        try (Connection conn = createStreamedConnection(false)) {
+            try (PreparedStatement stmt = conn.prepareStatement("insert into PUBLIC.Person(\"id\", \"name\") " +
+                "values (?, ?)")) {
+                for (int i = 1; i <= 100; i++) {
+                    stmt.setInt(1, i);
+                    stmt.setString(2, nameForId(i));
+
+                    stmt.executeUpdate();
+                }
+            }
+        }
 
-        PreparedStatement stmt = conn.prepareStatement("insert into Integer(_key, _val) values (?, ?)");
+        U.sleep(500);
 
+        // Now let's check it's all there.
         for (int i = 1; i <= 100; i++) {
-            stmt.setInt(1, i);
-            stmt.setInt(2, i);
+            if (i % 10 != 0)
+                assertEquals(nameForId(i), nameForIdInCache(i));
+            else // All that divides by 10 evenly should point to numbers 100 times greater - see above
+                assertEquals(nameForId(i * 100), nameForIdInCache(i));
+        }
+    }
 
-            stmt.executeUpdate();
+    /**
+     * @throws Exception if failed.
+     */
+    public void testStreamedInsertWithoutColumnsList() throws Exception {
+        for (int i = 10; i <= 100; i += 10)
+            put(i, nameForId(i * 100));
+
+        try (Connection conn = createStreamedConnection(false)) {
+            try (PreparedStatement stmt = conn.prepareStatement("insert into PUBLIC.Person(\"id\", \"name\") " +
+                "values (?, ?)")) {
+                for (int i = 1; i <= 100; i++) {
+                    stmt.setInt(1, i);
+                    stmt.setString(2, nameForId(i));
+
+                    stmt.executeUpdate();
+                }
+            }
         }
 
-        // Closing connection makes it wait for streamer close
-        // and thus for data load completion as well
-        conn.close();
+        U.sleep(500);
 
         // Now let's check it's all there.
         for (int i = 1; i <= 100; i++) {
             if (i % 10 != 0)
-                assertEquals(i, grid(0).cache(DEFAULT_CACHE_NAME).get(i));
+                assertEquals(nameForId(i), nameForIdInCache(i));
             else // All that divides by 10 evenly should point to numbers 100 times greater - see above
-                assertEquals(i * 100, grid(0).cache(DEFAULT_CACHE_NAME).get(i));
+                assertEquals(nameForId(i * 100), nameForIdInCache(i));
         }
     }
 
@@ -159,27 +233,101 @@ public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
      * @throws Exception if failed.
      */
     public void testStreamedInsertWithOverwritesAllowed() throws Exception {
-        conn = createConnection(true);
-
         for (int i = 10; i <= 100; i += 10)
-            ignite(0).cache(DEFAULT_CACHE_NAME).put(i, i * 100);
-
-        PreparedStatement stmt = conn.prepareStatement("insert into Integer(_key, _val) values (?, ?)");
-
-        for (int i = 1; i <= 100; i++) {
-            stmt.setInt(1, i);
-            stmt.setInt(2, i);
-
-            stmt.executeUpdate();
+            put(i, nameForId(i * 100));
+
+        try (Connection conn = createStreamedConnection(true)) {
+            try (PreparedStatement stmt = conn.prepareStatement("insert into PUBLIC.Person(\"id\", \"name\") " +
+                "values (?, ?)")) {
+                for (int i = 1; i <= 100; i++) {
+                    stmt.setInt(1, i);
+                    stmt.setString(2, nameForId(i));
+
+                    stmt.executeUpdate();
+                }
+            }
         }
 
-        // Closing connection makes it wait for streamer close
-        // and thus for data load completion as well
-        conn.close();
+        U.sleep(500);
 
         // Now let's check it's all there.
         // i should point to i at all times as we've turned overwrites on above.
         for (int i = 1; i <= 100; i++)
-            assertEquals(i, grid(0).cache(DEFAULT_CACHE_NAME).get(i));
+            assertEquals(nameForId(i), nameForIdInCache(i));
+    }
+
+    /** */
+    public void testOnlyInsertsAllowed() {
+        assertStatementForbidden("CREATE TABLE PUBLIC.X (x int primary key, y int)");
+
+        assertStatementForbidden("CREATE INDEX idx_1 ON Person(name)");
+
+        assertStatementForbidden("SELECT * from Person");
+
+        assertStatementForbidden("insert into PUBLIC.Person(\"id\", \"name\") " +
+            "(select \"id\" + 1, CONCAT(\"name\", '1') from Person)");
+
+        assertStatementForbidden("DELETE from Person");
+
+        assertStatementForbidden("UPDATE Person SET \"name\" = 'name0'");
+
+        assertStatementForbidden("alter table Person add column y int");
+    }
+
+    /**
+     * @param sql Statement to check.
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    protected void assertStatementForbidden(String sql) {
+        GridTestUtils.assertThrows(null, new IgniteCallable<Object>() {
+            @Override public Object call() throws Exception {
+                try (Connection c = createStreamedConnection(false)) {
+                    try (PreparedStatement s = c.prepareStatement(sql)) {
+                        s.execute();
+                    }
+                }
+
+                return null;
+            }
+        }, SQLException.class,"Streaming mode supports only INSERT commands without subqueries.");
+    }
+
+    /**
+     * @return Person cache.
+     */
+    protected IgniteCache<Integer, Object> cache() {
+        return grid(0).cache("person");
+    }
+
+    /**
+     * @param id id of person to put.
+     * @param name name of person to put.
+     */
+    protected void put(int id, String name) {
+        BinaryObjectBuilder bldr = grid(0).binary().builder("Person");
+
+        bldr.setField("name", name);
+
+        cache().put(id, bldr.build());
+    }
+
+    /**
+     * @param id Person id.
+     * @return Default name for person w/given id.
+     */
+    protected String nameForId(int id) {
+        return "Person" + id;
+    }
+
+    /**
+     * @param id person id.
+     * @return Name for person with given id currently stored in cache.
+     */
+    protected String nameForIdInCache(int id) {
+        Object o = cache().withKeepBinary().get(id);
+
+        assertTrue(String.valueOf(o), o instanceof BinaryObject);
+
+        return ((BinaryObject)o).field("name");
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index 4530ae7..f3a94ce 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -42,6 +42,7 @@ import org.apache.ignite.jdbc.thin.JdbcThinBulkLoadTransactionalPartitionedNearS
 import org.apache.ignite.jdbc.thin.JdbcThinBulkLoadTransactionalPartitionedSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinBulkLoadTransactionalReplicatedSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinComplexDmlDdlSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinComplexQuerySelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinConnectionSSLTest;
 import org.apache.ignite.jdbc.thin.JdbcThinConnectionSelfTest;
@@ -55,8 +56,10 @@ import org.apache.ignite.jdbc.thin.JdbcThinDynamicIndexTransactionalReplicatedSe
 import org.apache.ignite.jdbc.thin.JdbcThinEmptyCacheSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinErrorsSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinInsertStatementSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinInsertStatementSkipReducerOnUpdateSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinLocalQueriesSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinMergeStatementSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinMergeStatementSkipReducerOnUpdateSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinMetadataSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinMissingLongArrayResultsTest;
 import org.apache.ignite.jdbc.thin.JdbcThinNoDefaultSchemaTest;
@@ -65,11 +68,9 @@ import org.apache.ignite.jdbc.thin.JdbcThinResultSetSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinSchemaCaseTest;
 import org.apache.ignite.jdbc.thin.JdbcThinSelectAfterAlterTable;
 import org.apache.ignite.jdbc.thin.JdbcThinStatementSelfTest;
-import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSelfTest;
-import org.apache.ignite.jdbc.thin.JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest;
-import org.apache.ignite.jdbc.thin.JdbcThinInsertStatementSkipReducerOnUpdateSelfTest;
-import org.apache.ignite.jdbc.thin.JdbcThinMergeStatementSkipReducerOnUpdateSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinStreamingSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinTcpIoTest;
+import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinWalModeChangeSelfTest;
 
@@ -121,9 +122,11 @@ public class IgniteJdbcDriverTestSuite extends TestSuite {
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcStatementBatchingSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcErrorsSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcStreamingToPublicCacheTest.class));
+        suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcNoCacheStreamingSelfTest.class));
 
         suite.addTest(new TestSuite(JdbcBlobTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcStreamingSelfTest.class));
+        suite.addTest(new TestSuite(JdbcThinStreamingSelfTest.class));
 
         // DDL tests.
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcDynamicIndexAtomicPartitionedNearSelfTest.class));

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinStreamingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinStreamingSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinStreamingSelfTest.java
new file mode 100644
index 0000000..3c36f54
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinStreamingSelfTest.java
@@ -0,0 +1,486 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc.thin;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.internal.jdbc2.JdbcStreamingSelfTest;
+import org.apache.ignite.internal.processors.query.GridQueryCancel;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.SqlClientContext;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Tests for streaming via thin driver.
+ */
+public class JdbcThinStreamingSelfTest extends JdbcStreamingSelfTest {
+    /** */
+    private int batchSize = 17;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        GridQueryProcessor.idxCls = IndexingWithContext.class;
+
+        super.beforeTestsStarted();
+
+        batchSize = 17;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        try (Connection c = createOrdinaryConnection()) {
+            execute(c, "DROP TABLE PUBLIC.T IF EXISTS");
+        }
+
+        IndexingWithContext.cliCtx = null;
+
+        super.afterTest();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected Connection createStreamedConnection(boolean allowOverwrite, long flushFreq) throws Exception {
+        Connection c = JdbcThinAbstractSelfTest.connect(grid(0), null       );
+
+        execute(c, "SET STREAMING 1 BATCH_SIZE " + batchSize + " ALLOW_OVERWRITE " + (allowOverwrite ? 1 : 0) +
+            " PER_NODE_BUFFER_SIZE 1000 FLUSH_FREQUENCY " + flushFreq);
+
+        return c;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected Connection createOrdinaryConnection() throws SQLException {
+        return JdbcThinAbstractSelfTest.connect(grid(0), null);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testStreamedBatchedInsert() throws Exception {
+        for (int i = 10; i <= 100; i += 10)
+            put(i, nameForId(i * 100));
+
+        try (Connection conn = createStreamedConnection(false)) {
+            assertStreamingState(true);
+
+            try (PreparedStatement stmt = conn.prepareStatement("insert into Person(\"id\", \"name\") values (?, ?), " +
+                "(?, ?)")) {
+                for (int i = 1; i <= 100; i+= 2) {
+                    stmt.setInt(1, i);
+                    stmt.setString(2, nameForId(i));
+                    stmt.setInt(3, i + 1);
+                    stmt.setString(4, nameForId(i + 1));
+
+                    stmt.addBatch();
+                }
+
+                stmt.executeBatch();
+            }
+        }
+
+        U.sleep(500);
+
+        // Now let's check it's all there.
+        for (int i = 1; i <= 100; i++) {
+            if (i % 10 != 0)
+                assertEquals(nameForId(i), nameForIdInCache(i));
+            else // All that divides by 10 evenly should point to numbers 100 times greater - see above
+                assertEquals(nameForId(i * 100), nameForIdInCache(i));
+        }
+    }
+
+    /**
+     * @throws SQLException if failed.
+     */
+    public void testSimultaneousStreaming() throws Exception {
+        try (Connection anotherConn = createOrdinaryConnection()) {
+            execute(anotherConn, "CREATE TABLE PUBLIC.T(x int primary key, y int) WITH " +
+                "\"cache_name=T,wrap_value=false\"");
+        }
+
+        // Timeout to let connection close be handled on server side.
+        U.sleep(500);
+
+        try (Connection conn = createStreamedConnection(false, 10000)) {
+            assertStreamingState(true);
+
+            PreparedStatement firstStmt = conn.prepareStatement("insert into Person(\"id\", \"name\") values (?, ?)");
+
+            PreparedStatement secondStmt = conn.prepareStatement("insert into PUBLIC.T(x, y) values (?, ?)");
+
+            try {
+                for (int i = 1; i <= 10; i++) {
+                    firstStmt.setInt(1, i);
+                    firstStmt.setString(2, nameForId(i));
+
+                    firstStmt.executeUpdate();
+                }
+
+                for (int i = 51; i <= 67; i++) {
+                    secondStmt.setInt(1, i);
+                    secondStmt.setInt(2, i);
+
+                    secondStmt.executeUpdate();
+                }
+
+                for (int i = 11; i <= 50; i++) {
+                    firstStmt.setInt(1, i);
+                    firstStmt.setString(2, nameForId(i));
+
+                    firstStmt.executeUpdate();
+                }
+
+                for (int i = 68; i <= 100; i++) {
+                    secondStmt.setInt(1, i);
+                    secondStmt.setInt(2, i);
+
+                    secondStmt.executeUpdate();
+                }
+
+                assertCacheEmpty();
+
+                SqlClientContext cliCtx = sqlClientContext();
+
+                HashMap<String, IgniteDataStreamer<?, ?>> streamers = U.field(cliCtx, "streamers");
+
+                assertEquals(2, streamers.size());
+
+                assertEqualsCollections(new HashSet<>(Arrays.asList("person", "T")), streamers.keySet());
+            }
+            finally {
+                U.closeQuiet(firstStmt);
+
+                U.closeQuiet(secondStmt);
+            }
+        }
+
+        // Let's wait a little so that all data arrives to destination - we can't intercept streamers' flush
+        // on connection close in any way.
+        U.sleep(1000);
+
+        // Now let's check it's all there.
+        for (int i = 1; i <= 50; i++)
+            assertEquals(nameForId(i), nameForIdInCache(i));
+
+        for (int i = 51; i <= 100; i++)
+            assertEquals(i, grid(0).cache("T").get(i));
+    }
+
+    /**
+     *
+     */
+    public void testStreamingWithMixedStatementTypes() throws Exception {
+        String prepStmtStr = "insert into Person(\"id\", \"name\") values (?, ?)";
+
+        String stmtStr = "insert into Person(\"id\", \"name\") values (%d, '%s')";
+
+        try (Connection conn = createStreamedConnection(false, 10000)) {
+            assertStreamingState(true);
+
+            PreparedStatement firstStmt = conn.prepareStatement(prepStmtStr);
+
+            Statement secondStmt = conn.createStatement();
+
+            try {
+                for (int i = 1; i <= 100; i++) {
+                    boolean usePrep = Math.random() > 0.5;
+
+                    boolean useBatch = Math.random() > 0.5;
+
+                    if (usePrep) {
+                        firstStmt.setInt(1, i);
+                        firstStmt.setString(2, nameForId(i));
+
+                        if (useBatch)
+                            firstStmt.addBatch();
+                        else
+                            firstStmt.execute();
+                    }
+                    else {
+                        String sql = String.format(stmtStr, i, nameForId(i));
+
+                        if (useBatch)
+                            secondStmt.addBatch(sql);
+                        else
+                            secondStmt.execute(sql);
+                    }
+                }
+            }
+            finally {
+                U.closeQuiet(firstStmt);
+
+                U.closeQuiet(secondStmt);
+            }
+        }
+
+        // Let's wait a little so that all data arrives to destination - we can't intercept streamers' flush
+        // on connection close in any way.
+        U.sleep(1000);
+
+        // Now let's check it's all there.
+        for (int i = 1; i <= 100; i++)
+            assertEquals(nameForId(i), nameForIdInCache(i));
+    }
+
+    /**
+     * @throws SQLException if failed.
+     */
+    public void testStreamingOffToOn() throws SQLException {
+        try (Connection conn = createOrdinaryConnection()) {
+            assertStreamingState(false);
+
+            execute(conn, "SET STREAMING 1");
+
+            assertStreamingState(true);
+        }
+    }
+
+    /**
+     * @throws SQLException if failed.
+     */
+    public void testStreamingOnToOff() throws Exception {
+        try (Connection conn = createStreamedConnection(false)) {
+            assertStreamingState(true);
+
+            execute(conn, "SET STREAMING off");
+
+            assertStreamingState(false);
+        }
+    }
+
+    /**
+     * @throws SQLException if failed.
+     */
+    public void testFlush() throws Exception {
+        try (Connection conn = createStreamedConnection(false, 10000)) {
+            assertStreamingState(true);
+
+            try (PreparedStatement stmt = conn.prepareStatement("insert into Person(\"id\", \"name\") values (?, ?)")) {
+                for (int i = 1; i <= 100; i++) {
+                    stmt.setInt(1, i);
+                    stmt.setString(2, nameForId(i));
+
+                    stmt.executeUpdate();
+                }
+            }
+
+            assertCacheEmpty();
+
+            execute(conn, "set streaming 0");
+
+            assertStreamingState(false);
+
+            U.sleep(500);
+
+            // Now let's check it's all there.
+            for (int i = 1; i <= 100; i++)
+                assertEquals(nameForId(i), nameForIdInCache(i));
+        }
+    }
+
+    /**
+     * @throws SQLException if failed.
+     */
+    public void testStreamingReEnabled() throws Exception {
+        try (Connection conn = createStreamedConnection(false, 10000)) {
+            assertStreamingState(true);
+
+            try (PreparedStatement stmt = conn.prepareStatement("insert into Person(\"id\", \"name\") values (?, ?)")) {
+                for (int i = 1; i <= 100; i++) {
+                    stmt.setInt(1, i);
+                    stmt.setString(2, nameForId(i));
+
+                    stmt.executeUpdate();
+                }
+            }
+
+            assertCacheEmpty();
+
+            execute(conn, "set streaming 1 batch_size 111 allow_overwrite 0 per_node_buffer_size 512 " +
+                "per_node_parallel_operations 4 flush_frequency 5000");
+
+            U.sleep(500);
+
+            assertEquals((Integer)111, U.field(conn, "streamBatchSize"));
+
+            SqlClientContext cliCtx = sqlClientContext();
+
+            assertTrue(cliCtx.isStream());
+
+            assertFalse(U.field(cliCtx, "streamAllowOverwrite"));
+
+            assertEquals((Integer)512, U.field(cliCtx, "streamNodeBufSize"));
+
+            assertEquals((Long)5000L, U.field(cliCtx, "streamFlushTimeout"));
+
+            assertEquals((Integer)4, U.field(cliCtx, "streamNodeParOps"));
+
+            // Now let's check it's all there - SET STREAMING 1 repeated call must also have caused flush.
+            for (int i = 1; i <= 100; i++)
+                assertEquals(nameForId(i), nameForIdInCache(i));
+        }
+    }
+
+    /**
+     *
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    public void testNonStreamedBatch() {
+        GridTestUtils.assertThrows(null, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                try (Connection conn = createOrdinaryConnection()) {
+                    try (Statement s = conn.createStatement()) {
+                        for (int i = 1; i <= 10; i++)
+                            s.addBatch(String.format("insert into Person(\"id\", \"name\")values (%d, '%s')", i,
+                                nameForId(i)));
+
+                        execute(conn, "SET STREAMING 1");
+
+                        s.addBatch(String.format("insert into Person(\"id\", \"name\")values (%d, '%s')", 11,
+                            nameForId(11)));
+                    }
+                }
+
+                return null;
+            }
+        }, SQLException.class, "Statement has non-empty batch (call executeBatch() or clearBatch() before " +
+            "enabling streaming).");
+    }
+
+    /**
+     *
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    public void testStreamingStatementInTheMiddleOfNonPreparedBatch() {
+        GridTestUtils.assertThrows(null, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                try (Connection conn = createOrdinaryConnection()) {
+                    try (Statement s = conn.createStatement()) {
+                        s.addBatch(String.format("insert into Person(\"id\", \"name\")values (%d, '%s')", 1,
+                            nameForId(1)));
+
+                        s.addBatch("SET STREAMING 1 FLUSH_FREQUENCY 10000");
+                    }
+                }
+
+                return null;
+            }
+        }, SQLException.class, "Streaming control commands must be executed explicitly");
+    }
+
+    /**
+     *
+     */
+    @SuppressWarnings("ThrowableNotThrown")
+    public void testBatchingSetStreamingStatement() {
+        GridTestUtils.assertThrows(null, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                try (Connection conn = createOrdinaryConnection()) {
+                    try (PreparedStatement s = conn.prepareStatement("SET STREAMING 1 FLUSH_FREQUENCY 10000")) {
+                        s.addBatch();
+                    }
+                }
+
+                return null;
+            }
+        }, SQLException.class, "Streaming control commands must be executed explicitly");
+    }
+
+    /**
+     * Check that there's nothing in cache.
+     */
+    private void assertCacheEmpty() {
+        assertEquals(0, cache().size(CachePeekMode.ALL));
+    }
+
+    /**
+     * @param conn Connection.
+     * @param sql Statement.
+     * @throws SQLException if failed.
+     */
+    private static void execute(Connection conn, String sql) throws SQLException {
+        try (Statement s = conn.createStatement()) {
+            s.execute(sql);
+        }
+    }
+
+    /**
+     * @return Active SQL client context.
+     */
+    private SqlClientContext sqlClientContext() {
+        assertNotNull(IndexingWithContext.cliCtx);
+
+        return IndexingWithContext.cliCtx;
+    }
+
+    /**
+     * Check that streaming state on target node is as expected.
+     * @param on Expected streaming state.
+     */
+    private void assertStreamingState(boolean on) {
+        SqlClientContext cliCtx = sqlClientContext();
+
+        assertEquals(on, cliCtx.isStream());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void assertStatementForbidden(String sql) {
+        batchSize = 1;
+
+        super.assertStatementForbidden(sql);
+    }
+
+    /**
+     *
+     */
+    private static final class IndexingWithContext extends IgniteH2Indexing {
+        /** Client context. */
+        static SqlClientContext cliCtx;
+
+        /** {@inheritDoc} */
+        @Override public List<Long> streamBatchedUpdateQuery(String schemaName, String qry, List<Object[]> params,
+            SqlClientContext cliCtx) throws IgniteCheckedException {
+            IndexingWithContext.cliCtx = cliCtx;
+
+            return super.streamBatchedUpdateQuery(schemaName, qry, params, cliCtx);
+        }
+
+        /** {@inheritDoc} */
+        @Override public List<FieldsQueryCursor<List<?>>> querySqlFields(String schemaName, SqlFieldsQuery qry,
+            @Nullable SqlClientContext cliCtx, boolean keepBinary, boolean failOnMultipleStmts,
+            GridQueryCancel cancel) {
+            IndexingWithContext.cliCtx = cliCtx;
+
+            return super.querySqlFields(schemaName, qry, cliCtx, keepBinary, failOnMultipleStmts, cancel);
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java
index ac9925d..779a022 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.jdbc.thin;
 
 import java.sql.Array;
+import java.sql.BatchUpdateException;
 import java.sql.Blob;
 import java.sql.CallableStatement;
 import java.sql.Clob;
@@ -33,17 +34,27 @@ import java.sql.SQLXML;
 import java.sql.Savepoint;
 import java.sql.Statement;
 import java.sql.Struct;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.Executor;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
 import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
 import org.apache.ignite.internal.processors.odbc.SqlStateCode;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBatchExecuteRequest;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBatchExecuteResult;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQuery;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryExecuteRequest;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcRequest;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcResponse;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcResult;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcStatementType;
 import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.sql.command.SqlCommand;
+import org.apache.ignite.internal.sql.command.SqlSetStreamingCommand;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.lang.IgniteProductVersion;
 
@@ -79,6 +90,9 @@ public class JdbcThinConnection implements Connection {
     /** Read-only flag. */
     private boolean readOnly;
 
+    /** Streaming flag. */
+    private volatile boolean stream;
+
     /** Current transaction holdability. */
     private int holdability;
 
@@ -94,6 +108,15 @@ public class JdbcThinConnection implements Connection {
     /** Connection properties. */
     private ConnectionProperties connProps;
 
+    /** Batch size for streaming. */
+    private int streamBatchSize;
+
+    /** Batch for streaming. */
+    private List<JdbcQuery> streamBatch;
+
+    /** Last added query to recognize batches. */
+    private String lastStreamQry;
+
     /**
      * Creates new connection.
      *
@@ -135,6 +158,86 @@ public class JdbcThinConnection implements Connection {
         }
     }
 
+    /**
+     * @return Whether this connection is streamed or not.
+     */
+    boolean isStream() {
+        return stream;
+    }
+
+    /**
+     * @param sql Statement.
+     * @param cmd Parsed form of {@code sql}.
+     * @throws SQLException if failed.
+     */
+    void executeNative(String sql, SqlCommand cmd) throws SQLException {
+        if (cmd instanceof SqlSetStreamingCommand) {
+            // If streaming is already on, we have to disable it first.
+            if (stream) {
+                // We have to send request regardless of actual batch size.
+                executeBatch(true);
+
+                stream = false;
+            }
+
+            boolean newVal = ((SqlSetStreamingCommand)cmd).isTurnOn();
+
+            // Actual ON, if needed.
+            if (newVal) {
+                sendRequest(new JdbcQueryExecuteRequest(JdbcStatementType.ANY_STATEMENT_TYPE,
+                    schema, 1, 1, sql, null));
+
+                streamBatchSize = ((SqlSetStreamingCommand)cmd).batchSize();
+
+                stream = true;
+            }
+        }
+        else
+            throw IgniteQueryErrorCode.createJdbcSqlException("Unsupported native statement: " + sql,
+                IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+    }
+
+    /**
+     * Add another query for batched execution.
+     * @param sql Query.
+     * @param args Arguments.
+     */
+    void addBatch(String sql, List<Object> args) throws SQLException {
+        boolean newQry = (args == null || !F.eq(lastStreamQry, sql));
+
+        // Providing null as SQL here allows for recognizing subbatches on server and handling them more efficiently.
+        JdbcQuery q  = new JdbcQuery(newQry ? sql : null, args != null ? args.toArray() : null);
+
+        if (streamBatch == null)
+            streamBatch = new ArrayList<>(streamBatchSize);
+
+        streamBatch.add(q);
+
+        // Null args means "addBatch(String)" was called on non-prepared Statement,
+        // we don't want to remember its query string.
+        lastStreamQry = (args != null ? sql : null);
+
+        if (streamBatch.size() == streamBatchSize)
+            executeBatch(false);
+    }
+
+    /**
+     * @param lastBatch Whether open data streamers must be flushed and closed after this batch.
+     * @throws SQLException if failed.
+     */
+    private void executeBatch(boolean lastBatch) throws SQLException {
+        JdbcBatchExecuteResult res = sendRequest(new JdbcBatchExecuteRequest(schema, streamBatch, lastBatch));
+
+        streamBatch = null;
+
+        lastStreamQry = null;
+
+        if (res.errorCode() != ClientListenerResponse.STATUS_SUCCESS) {
+            throw new BatchUpdateException(res.errorMessage(), IgniteQueryErrorCode.codeToSqlState(res.errorCode()),
+                res.errorCode(), res.updateCounts());
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public Statement createStatement() throws SQLException {
         return createStatement(TYPE_FORWARD_ONLY, CONCUR_READ_ONLY, HOLD_CURSORS_OVER_COMMIT);
@@ -277,6 +380,15 @@ public class JdbcThinConnection implements Connection {
         if (isClosed())
             return;
 
+        if (!F.isEmpty(streamBatch)) {
+            try {
+                executeBatch(true);
+            }
+            catch (SQLException e) {
+                LOG.log(Level.WARNING, "Exception during batch send on streamed connection close", e);
+            }
+        }
+
         closed = true;
 
         cliIo.close();

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java
index 23d3bbe..d18788c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java
@@ -39,8 +39,8 @@ import java.sql.Time;
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Calendar;
-import org.apache.ignite.internal.processors.odbc.SqlStateCode;
 import org.apache.ignite.internal.processors.odbc.SqlListenerUtils;
+import org.apache.ignite.internal.processors.odbc.SqlStateCode;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcMetaParamsRequest;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcMetaParamsResult;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQuery;
@@ -262,13 +262,23 @@ public class JdbcThinPreparedStatement extends JdbcThinStatement implements Prep
     @Override public void addBatch() throws SQLException {
         ensureNotClosed();
 
-        if (batch == null) {
-            batch = new ArrayList<>();
+        checkStatementEligibleForBatching(sql);
+
+        checkStatementBatchEmpty();
+
+        batchSize++;
+
+        if (conn.isStream())
+            conn.addBatch(sql, args);
+        else {
+            if (batch == null) {
+                batch = new ArrayList<>();
 
-            batch.add(new JdbcQuery(sql, args.toArray(new Object[args.size()])));
+                batch.add(new JdbcQuery(sql, args.toArray(new Object[args.size()])));
+            }
+            else
+                batch.add(new JdbcQuery(null, args.toArray(new Object[args.size()])));
         }
-        else
-            batch.add(new JdbcQuery(null, args.toArray(new Object[args.size()])));
 
         args = null;
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java
index 9c41804..480edfa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java
@@ -38,14 +38,21 @@ import org.apache.ignite.internal.processors.odbc.SqlStateCode;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBatchExecuteRequest;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBatchExecuteResult;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBulkLoadAckResult;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBulkLoadBatchRequest;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQuery;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryExecuteMultipleStatementsResult;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryExecuteRequest;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryExecuteResult;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcResult;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcResultInfo;
-import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBulkLoadBatchRequest;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcStatementType;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
+import org.apache.ignite.internal.sql.SqlKeyword;
+import org.apache.ignite.internal.sql.SqlParseException;
+import org.apache.ignite.internal.sql.SqlParser;
+import org.apache.ignite.internal.sql.command.SqlCommand;
+import org.apache.ignite.internal.sql.command.SqlSetStreamingCommand;
+import org.apache.ignite.internal.util.typedef.F;
 
 import static java.sql.ResultSet.CONCUR_READ_ONLY;
 import static java.sql.ResultSet.FETCH_FORWARD;
@@ -79,6 +86,9 @@ public class JdbcThinStatement implements Statement {
     /** Result set  holdability*/
     private final int resHoldability;
 
+    /** Batch size to keep track of number of items to return as fake update counters for executeBatch. */
+    protected int batchSize;
+
     /** Batch. */
     protected List<JdbcQuery> batch;
 
@@ -119,6 +129,40 @@ public class JdbcThinStatement implements Statement {
     }
 
     /**
+     * @param sql Query.
+     * @return Native {@link SqlCommand}, or {@code null} if parsing was not successful.
+     */
+    private SqlCommand tryParseNative(String sql) {
+        try {
+            return new SqlParser(schema, sql).nextCommand();
+        }
+        catch (SqlParseException e) {
+            return null;
+        }
+    }
+
+    /**
+     * @param sql Query.
+     * @return Whether it's worth trying to parse this statement on the client.
+     */
+    private static boolean isEligibleForNativeParsing(String sql) {
+        if (F.isEmpty(sql))
+            return false;
+
+        sql = sql.toUpperCase();
+
+        int setPos = sql.indexOf(SqlKeyword.SET);
+
+        if (setPos == -1)
+            return false;
+
+        int streamingPos = sql.indexOf(SqlKeyword.STREAMING);
+
+        // There must be at least one symbol between SET and STREAMING.
+        return streamingPos - setPos - SqlKeyword.SET.length() >= 1;
+    }
+
+    /**
      * @param stmtType Expected statement type.
      * @param sql Sql query.
      * @param args Query parameters.
@@ -133,6 +177,36 @@ public class JdbcThinStatement implements Statement {
         if (sql == null || sql.isEmpty())
             throw new SQLException("SQL query is empty.");
 
+        checkStatementBatchEmpty();
+
+        SqlCommand nativeCmd = null;
+
+        if (stmtType != JdbcStatementType.SELECT_STATEMENT_TYPE && isEligibleForNativeParsing(sql))
+            nativeCmd = tryParseNative(sql);
+
+        if (nativeCmd != null) {
+            conn.executeNative(sql, nativeCmd);
+
+            resultSets = Collections.singletonList(resultSetForUpdate(0));
+
+            // If this command should be executed as native one, we do not treat it
+            // as an ordinary batch citizen.
+            return;
+        }
+
+        if (conn.isStream()) {
+            if (stmtType == JdbcStatementType.SELECT_STATEMENT_TYPE)
+                throw new SQLException("executeQuery() method is not allowed in streaming mode.",
+                    SqlStateCode.INTERNAL_ERROR,
+                    IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+
+            conn.addBatch(sql, args);
+
+            resultSets = Collections.singletonList(resultSetForUpdate(0));
+
+            return;
+        }
+
         JdbcResult res0 = conn.sendRequest(new JdbcQueryExecuteRequest(stmtType, schema, pageSize,
             maxRows, sql, args == null ? null : args.toArray(new Object[args.size()])));
 
@@ -158,11 +232,8 @@ public class JdbcThinStatement implements Statement {
             boolean firstRes = true;
 
             for(JdbcResultInfo rsInfo : resInfos) {
-                if (!rsInfo.isQuery()) {
-                    resultSets.add(new JdbcThinResultSet(this, -1, pageSize,
-                        true, Collections.<List<Object>>emptyList(), false,
-                        conn.autoCloseServerCursor(), rsInfo.updateCount(), closeOnCompletion));
-                }
+                if (!rsInfo.isQuery())
+                    resultSets.add(resultSetForUpdate(rsInfo.updateCount()));
                 else {
                     if (firstRes) {
                         firstRes = false;
@@ -186,6 +257,26 @@ public class JdbcThinStatement implements Statement {
     }
 
     /**
+     * Check that user has not added anything to this statement's batch prior to turning streaming on.
+     * @throws SQLException if failed.
+     */
+    void checkStatementBatchEmpty() throws SQLException {
+        if (conn.isStream() && !F.isEmpty(batch))
+            throw new IgniteSQLException("Statement has non-empty batch (call executeBatch() or clearBatch() " +
+                "before enabling streaming).", IgniteQueryErrorCode.UNSUPPORTED_OPERATION).toJdbcException();
+    }
+
+    /**
+     * @param cnt Update counter.
+     * @return Result set for given update counter.
+     */
+    private JdbcThinResultSet resultSetForUpdate(long cnt) {
+        return new JdbcThinResultSet(this, -1, pageSize,
+            true, Collections.<List<Object>>emptyList(), false,
+            conn.autoCloseServerCursor(), cnt, closeOnCompletion);
+    }
+
+    /**
      * Sends a file to server in batches via multiple {@link JdbcBulkLoadBatchRequest}s.
      *
      * @param cmdRes Result of invoking COPY command: contains server-parsed
@@ -469,16 +560,50 @@ public class JdbcThinStatement implements Statement {
     @Override public void addBatch(String sql) throws SQLException {
         ensureNotClosed();
 
+        checkStatementEligibleForBatching(sql);
+
+        checkStatementBatchEmpty();
+
+        batchSize++;
+
+        if (conn.isStream()) {
+            conn.addBatch(sql, null);
+
+            return;
+        }
+
         if (batch == null)
             batch = new ArrayList<>();
 
         batch.add(new JdbcQuery(sql, null));
     }
 
+    /**
+     * Check that we're not trying to add to connection's batch a native command (it should be executed explicitly).
+     * @param sql SQL command.
+     * @throws SQLException if there's an attempt to add a native command to JDBC batch.
+     */
+    void checkStatementEligibleForBatching(String sql) throws SQLException {
+        SqlCommand nativeCmd = null;
+
+        if (isEligibleForNativeParsing(sql))
+            nativeCmd = tryParseNative(sql);
+
+        if (nativeCmd != null) {
+            assert nativeCmd instanceof SqlSetStreamingCommand;
+
+            throw new SQLException("Streaming control commands must be executed explicitly - " +
+                "either via Statement.execute(String), or via using prepared statements.",
+                SqlStateCode.UNSUPPORTED_OPERATION);
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public void clearBatch() throws SQLException {
         ensureNotClosed();
 
+        batchSize = 0;
+
         batch = null;
     }
 
@@ -488,11 +613,21 @@ public class JdbcThinStatement implements Statement {
 
         closeResults();
 
-        if (batch == null || batch.isEmpty())
+        checkStatementBatchEmpty();
+
+        if (conn.isStream()) {
+            int[] res = new int[batchSize];
+
+            batchSize = 0;
+
+            return res;
+        }
+
+        if (F.isEmpty(batch))
             throw new SQLException("Batch is empty.");
 
         try {
-            JdbcBatchExecuteResult res = conn.sendRequest(new JdbcBatchExecuteRequest(conn.getSchema(), batch));
+            JdbcBatchExecuteResult res = conn.sendRequest(new JdbcBatchExecuteRequest(conn.getSchema(), batch, false));
 
             if (res.errorCode() != ClientListenerResponse.STATUS_SUCCESS) {
                 throw new BatchUpdateException(res.errorMessage(), IgniteQueryErrorCode.codeToSqlState(res.errorCode()),
@@ -502,6 +637,8 @@ public class JdbcThinStatement implements Statement {
             return res.updateCounts();
         }
         finally {
+            batchSize = 0;
+
             batch = null;
         }
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
index 7aa6c33..ba7258a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
@@ -37,12 +37,14 @@ import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
 import org.apache.ignite.internal.processors.odbc.ClientListenerRequest;
 import org.apache.ignite.internal.processors.odbc.SqlStateCode;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBatchExecuteRequest;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQuery;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryCloseRequest;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryFetchRequest;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryMetadataRequest;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcRequest;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcResponse;
 import org.apache.ignite.internal.util.ipc.loopback.IpcClientTcpEndpoint;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteProductVersion;
 
@@ -298,7 +300,8 @@ public class JdbcThinTcpIo {
 
             ClientListenerProtocolVersion srvProtocolVer = ClientListenerProtocolVersion.create(maj, min, maintenance);
 
-            if (VER_2_3_0.equals(srvProtocolVer) || VER_2_1_5.equals(srvProtocolVer))
+            if (VER_2_4_0.equals(srvProtocolVer) || VER_2_3_0.equals(srvProtocolVer) ||
+                VER_2_1_5.equals(srvProtocolVer))
                 handshake(srvProtocolVer);
             else if (VER_2_1_0.equals(srvProtocolVer))
                 handshake_2_1_0();
@@ -391,9 +394,12 @@ public class JdbcThinTcpIo {
         int cap;
 
         if (req instanceof JdbcBatchExecuteRequest) {
-            int cnt = Math.min(MAX_BATCH_QRY_CNT, ((JdbcBatchExecuteRequest)req).queries().size());
+            List<JdbcQuery> qrys = ((JdbcBatchExecuteRequest)req).queries();
 
-            cap = cnt * DYNAMIC_SIZE_MSG_CAP;
+            int cnt = !F.isEmpty(qrys) ? Math.min(MAX_BATCH_QRY_CNT, qrys.size()) : 0;
+
+            // One additional byte for last batch flag.
+            cap = cnt * DYNAMIC_SIZE_MSG_CAP + 1;
         }
         else if (req instanceof JdbcQueryCloseRequest)
             cap = QUERY_CLOSE_MSG_SIZE;

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
index b51e0b9..f61ccf9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
@@ -60,6 +60,7 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
 import org.apache.ignite.internal.processors.odbc.SqlStateCode;
 import org.apache.ignite.internal.processors.query.GridQueryIndexing;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
 import org.apache.ignite.internal.processors.query.QueryUtils;
 import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -612,10 +613,11 @@ public class JdbcConnection implements Connection {
 
             PreparedStatement nativeStmt = prepareNativeStatement(sql);
 
-            if (!idx.isInsertStatement(nativeStmt)) {
-                throw new SQLException("Only INSERT operations are supported in streaming mode",
-                    SqlStateCode.INTERNAL_ERROR,
-                    IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+            try {
+                idx.checkStatementStreamable(nativeStmt);
+            }
+            catch (IgniteSQLException e) {
+                throw e.toJdbcException();
             }
 
             IgniteDataStreamer streamer = ignite().dataStreamer(cacheName);

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java
index 408f089..25f55f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java
@@ -55,7 +55,7 @@ class JdbcStreamedPreparedStatement extends JdbcPreparedStatement {
 
     /** {@inheritDoc} */
     @Override protected void execute0(String sql, Boolean isQuery) throws SQLException {
-        assert isQuery != null && !isQuery;
+        assert isQuery == null || !isQuery;
 
         long updCnt = conn.ignite().context().query().streamUpdateQuery(conn.cacheName(), conn.schemaName(),
             streamer, sql, getArgs());

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index a834022..c5d68b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -629,7 +629,7 @@ public class IgniteCacheProxyImpl<K, V> extends AsyncSupportAdapter<IgniteCache<
 
             boolean keepBinary = opCtxCall != null && opCtxCall.isKeepBinary();
 
-            return ctx.kernalContext().query().querySqlFields(ctx, qry, keepBinary, false);
+            return ctx.kernalContext().query().querySqlFields(ctx, qry, null, keepBinary, false);
         }
         catch (Exception e) {
             if (e instanceof CacheException)
@@ -662,7 +662,7 @@ public class IgniteCacheProxyImpl<K, V> extends AsyncSupportAdapter<IgniteCache<
 
             if (qry instanceof SqlFieldsQuery)
                 return (FieldsQueryCursor<R>)ctx.kernalContext().query().querySqlFields(ctx, (SqlFieldsQuery)qry,
-                    keepBinary, true).get(0);
+                    null, keepBinary, true).get(0);
 
             if (qry instanceof ScanQuery)
                 return query((ScanQuery)qry, null, projection(qry.isLocal()));

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java
index 25e1049..73fd04f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.odbc.jdbc;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.ignite.binary.BinaryObjectException;
@@ -39,6 +40,12 @@ public class JdbcBatchExecuteRequest extends JdbcRequest {
     private List<JdbcQuery> queries;
 
     /**
+     * Last stream batch flag - whether open streamers on current connection
+     * must be flushed and closed after this batch.
+     */
+    private boolean lastStreamBatch;
+
+    /**
      * Default constructor.
      */
     public JdbcBatchExecuteRequest() {
@@ -49,13 +56,14 @@ public class JdbcBatchExecuteRequest extends JdbcRequest {
      * @param schemaName Schema name.
      * @param queries Queries.
      */
-    public JdbcBatchExecuteRequest(String schemaName, List<JdbcQuery> queries) {
+    public JdbcBatchExecuteRequest(String schemaName, List<JdbcQuery> queries, boolean lastStreamBatch) {
         super(BATCH_EXEC);
 
-        assert !F.isEmpty(queries);
+        assert lastStreamBatch || !F.isEmpty(queries);
 
         this.schemaName = schemaName;
         this.queries = queries;
+        this.lastStreamBatch = lastStreamBatch;
     }
 
     /**
@@ -72,15 +80,29 @@ public class JdbcBatchExecuteRequest extends JdbcRequest {
         return queries;
     }
 
+    /**
+     * @return Last stream batch flag.
+     */
+    public boolean isLastStreamBatch() {
+        return lastStreamBatch;
+    }
+
     /** {@inheritDoc} */
     @Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
         super.writeBinary(writer);
 
         writer.writeString(schemaName);
-        writer.writeInt(queries.size());
 
-        for (JdbcQuery q : queries)
-            q.writeBinary(writer);
+        if (!F.isEmpty(queries)) {
+            writer.writeInt(queries.size());
+
+            for (JdbcQuery q : queries)
+                q.writeBinary(writer);
+        }
+        else
+            writer.writeInt(0);
+
+        writer.writeBoolean(lastStreamBatch);
     }
 
     /** {@inheritDoc} */
@@ -100,6 +122,14 @@ public class JdbcBatchExecuteRequest extends JdbcRequest {
 
             queries.add(qry);
         }
+
+        try {
+            if (reader.available() > 0)
+                lastStreamBatch = reader.readBoolean();
+        }
+        catch (IOException e) {
+            throw new BinaryObjectException(e);
+        }
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
index 5841a4d..d8f82f5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
@@ -28,7 +28,7 @@ import org.apache.ignite.internal.processors.odbc.ClientListenerRequestHandler;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
 
 /**
- * ODBC Connection Context.
+ * JDBC Connection Context.
  */
 public class JdbcConnectionContext implements ClientListenerConnectionContext {
     /** Version 2.1.0. */
@@ -38,13 +38,16 @@ public class JdbcConnectionContext implements ClientListenerConnectionContext {
     private static final ClientListenerProtocolVersion VER_2_1_5 = ClientListenerProtocolVersion.create(2, 1, 5);
 
     /** Version 2.3.1: added "multiple statements query" feature. */
-    public static final ClientListenerProtocolVersion VER_2_3_0 = ClientListenerProtocolVersion.create(2, 3, 0);
+    static final ClientListenerProtocolVersion VER_2_3_0 = ClientListenerProtocolVersion.create(2, 3, 0);
 
     /** Version 2.4.0: adds default values for columns feature. */
-    public static final ClientListenerProtocolVersion VER_2_4_0 = ClientListenerProtocolVersion.create(2, 4, 0);
+    static final ClientListenerProtocolVersion VER_2_4_0 = ClientListenerProtocolVersion.create(2, 4, 0);
+
+    /** Version 2.5.0. */
+    private static final ClientListenerProtocolVersion VER_2_5_0 = ClientListenerProtocolVersion.create(2, 5, 0);
 
     /** Current version. */
-    private static final ClientListenerProtocolVersion CURRENT_VER = VER_2_4_0;
+    private static final ClientListenerProtocolVersion CURRENT_VER = VER_2_5_0;
 
     /** Supported versions. */
     private static final Set<ClientListenerProtocolVersion> SUPPORTED_VERS = new HashSet<>();
@@ -66,6 +69,8 @@ public class JdbcConnectionContext implements ClientListenerConnectionContext {
 
     static {
         SUPPORTED_VERS.add(CURRENT_VER);
+        SUPPORTED_VERS.add(VER_2_5_0);
+        SUPPORTED_VERS.add(VER_2_4_0);
         SUPPORTED_VERS.add(VER_2_3_0);
         SUPPORTED_VERS.add(VER_2_1_5);
         SUPPORTED_VERS.add(VER_2_1_0);

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
index 59fc06b..baf4ec5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
@@ -53,6 +53,7 @@ import org.apache.ignite.internal.processors.query.GridQueryProperty;
 import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
 import org.apache.ignite.internal.processors.query.IgniteSQLException;
 import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.processors.query.SqlClientContext;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
@@ -88,6 +89,9 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
     /** Kernel context. */
     private final GridKernalContext ctx;
 
+    /** Client context. */
+    private final SqlClientContext cliCtx;
+
     /** Logger. */
     private final IgniteLogger log;
 
@@ -103,24 +107,6 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
     /** Current bulk load processors. */
     private final ConcurrentHashMap<Long, JdbcBulkLoadProcessor> bulkLoadRequests = new ConcurrentHashMap<>();
 
-    /** Distributed joins flag. */
-    private final boolean distributedJoins;
-
-    /** Enforce join order flag. */
-    private final boolean enforceJoinOrder;
-
-    /** Collocated flag. */
-    private final boolean collocated;
-
-    /** Replicated only flag. */
-    private final boolean replicatedOnly;
-
-    /** Lazy query execution flag. */
-    private final boolean lazy;
-
-    /** Skip reducer on update flag. */
-    private final boolean skipReducerOnUpdate;
-
     /** Automatic close of cursors. */
     private final boolean autoCloseCursors;
 
@@ -129,7 +115,6 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
 
     /**
      * Constructor.
-     *
      * @param ctx Context.
      * @param busyLock Shutdown latch.
      * @param maxCursors Maximum allowed cursors.
@@ -147,15 +132,20 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
         boolean autoCloseCursors, boolean lazy, boolean skipReducerOnUpdate,
         ClientListenerProtocolVersion protocolVer) {
         this.ctx = ctx;
+
+        this.cliCtx = new SqlClientContext(
+            ctx,
+            distributedJoins,
+            enforceJoinOrder,
+            collocated,
+            replicatedOnly,
+            lazy,
+            skipReducerOnUpdate
+        );
+
         this.busyLock = busyLock;
         this.maxCursors = maxCursors;
-        this.distributedJoins = distributedJoins;
-        this.enforceJoinOrder = enforceJoinOrder;
-        this.collocated = collocated;
-        this.replicatedOnly = replicatedOnly;
         this.autoCloseCursors = autoCloseCursors;
-        this.lazy = lazy;
-        this.skipReducerOnUpdate = skipReducerOnUpdate;
         this.protocolVer = protocolVer;
 
         log = ctx.log(getClass());
@@ -301,6 +291,8 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
                 }
 
                 bulkLoadRequests.clear();
+
+                U.close(cliCtx, log);
             }
             finally {
                 busyLock.leaveBusy();
@@ -326,6 +318,8 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
 
         long qryId = QRY_ID_GEN.getAndIncrement();
 
+        assert !cliCtx.isStream();
+
         try {
             String sql = req.sqlQuery();
 
@@ -347,17 +341,17 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
 
                     qry = new SqlFieldsQueryEx(sql, false);
 
-                    if (skipReducerOnUpdate)
+                    if (cliCtx.isSkipReducerOnUpdate())
                         ((SqlFieldsQueryEx)qry).setSkipReducerOnUpdate(true);
             }
 
             qry.setArgs(req.arguments());
 
-            qry.setDistributedJoins(distributedJoins);
-            qry.setEnforceJoinOrder(enforceJoinOrder);
-            qry.setCollocated(collocated);
-            qry.setReplicatedOnly(replicatedOnly);
-            qry.setLazy(lazy);
+            qry.setDistributedJoins(cliCtx.isDistributedJoins());
+            qry.setEnforceJoinOrder(cliCtx.isEnforceJoinOrder());
+            qry.setCollocated(cliCtx.isCollocated());
+            qry.setReplicatedOnly(cliCtx.isReplicatedOnly());
+            qry.setLazy(cliCtx.isLazy());
 
             if (req.pageSize() <= 0)
                 return new JdbcResponse(IgniteQueryErrorCode.UNKNOWN, "Invalid fetch size: " + req.pageSize());
@@ -371,7 +365,7 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
 
             qry.setSchema(schemaName);
 
-            List<FieldsQueryCursor<List<?>>> results = ctx.query().querySqlFields(qry, true,
+            List<FieldsQueryCursor<List<?>>> results = ctx.query().querySqlFields(null, qry, cliCtx, true,
                 protocolVer.compareTo(VER_2_3_0) < 0);
 
             FieldsQueryCursor<List<?>> fieldsCur = results.get(0);
@@ -569,11 +563,11 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
 
                 qry = new SqlFieldsQueryEx(q.sql(), false);
 
-                qry.setDistributedJoins(distributedJoins);
-                qry.setEnforceJoinOrder(enforceJoinOrder);
-                qry.setCollocated(collocated);
-                qry.setReplicatedOnly(replicatedOnly);
-                qry.setLazy(lazy);
+                qry.setDistributedJoins(cliCtx.isDistributedJoins());
+                qry.setEnforceJoinOrder(cliCtx.isEnforceJoinOrder());
+                qry.setCollocated(cliCtx.isCollocated());
+                qry.setReplicatedOnly(cliCtx.isReplicatedOnly());
+                qry.setLazy(cliCtx.isLazy());
 
                 qry.setSchema(schemaName);
             }
@@ -586,6 +580,9 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
         if (qry != null)
             executeBatchedQuery(qry, updCntsAcc, firstErr);
 
+        if (req.isLastStreamBatch())
+            cliCtx.disableStreaming();
+
         int updCnts[] = U.toIntArray(updCntsAcc);
 
         if (firstErr.isEmpty())
@@ -601,10 +598,21 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
      * @param updCntsAcc Per query rows updates counter.
      * @param firstErr First error data - code and message.
      */
+    @SuppressWarnings("ForLoopReplaceableByForEach")
     private void executeBatchedQuery(SqlFieldsQueryEx qry, List<Integer> updCntsAcc,
         IgniteBiTuple<Integer, String> firstErr) {
         try {
-            List<FieldsQueryCursor<List<?>>> qryRes = ctx.query().querySqlFields(qry, true, true);
+            if (cliCtx.isStream()) {
+                List<Long> cnt = ctx.query().streamBatchedUpdateQuery(qry.getSchema(), cliCtx, qry.getSql(),
+                    qry.batchedArguments());
+
+                for (int i = 0; i < cnt.size(); i++)
+                    updCntsAcc.add(cnt.get(i).intValue());
+
+                return;
+            }
+
+            List<FieldsQueryCursor<List<?>>> qryRes = ctx.query().querySqlFields(null, qry, cliCtx, true, true);
 
             for (FieldsQueryCursor<List<?>> cur : qryRes) {
                 if (cur instanceof BulkLoadContextCursor)

http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index 6b425a1..dedd075 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@ -76,33 +76,46 @@ public interface GridQueryIndexing {
      * Detect whether SQL query should be executed in distributed or local manner and execute it.
      * @param schemaName Schema name.
      * @param qry Query.
+     * @param cliCtx Client context.
      * @param keepBinary Keep binary flag.
      * @param failOnMultipleStmts Whether an exception should be thrown for multiple statements query.
-     * @param cancel Query cancel state handler.
-     * @return Cursor.
+     * @param cancel Query cancel state handler.    @return Cursor.
      */
-    public List<FieldsQueryCursor<List<?>>> querySqlFields(String schemaName, SqlFieldsQuery qry, boolean keepBinary,
-        boolean failOnMultipleStmts, GridQueryCancel cancel);
+    public List<FieldsQueryCursor<List<?>>> querySqlFields(String schemaName, SqlFieldsQuery qry,
+        SqlClientContext cliCtx, boolean keepBinary, boolean failOnMultipleStmts, GridQueryCancel cancel);
 
     /**
-     * Perform a MERGE statement using data streamer as receiver.
+     * Execute an INSERT statement using data streamer as receiver.
      *
      * @param schemaName Schema name.
      * @param qry Query.
      * @param params Query parameters.
      * @param streamer Data streamer to feed data to.
-     * @return Query result.
+     * @return Update counter.
      * @throws IgniteCheckedException If failed.
      */
     public long streamUpdateQuery(String schemaName, String qry, @Nullable Object[] params,
         IgniteDataStreamer<?, ?> streamer) throws IgniteCheckedException;
 
     /**
+     * Execute a batched INSERT statement using data streamer as receiver.
+     *
+     * @param schemaName Schema name.
+     * @param qry Query.
+     * @param params Query parameters.
+     * @param cliCtx Client connection context.
+     * @return Update counters.
+     * @throws IgniteCheckedException If failed.
+     */
+    public List<Long> streamBatchedUpdateQuery(String schemaName, String qry, List<Object[]> params,
+        SqlClientContext cliCtx) throws IgniteCheckedException;
+
+    /**
      * Executes regular query.
      *
      * @param schemaName Schema name.
      * @param cacheName Cache name.
-     *@param qry Query.
+     * @param qry Query.
      * @param filter Cache name and key filter.
      * @param keepBinary Keep binary flag.    @return Cursor.
      */
@@ -313,12 +326,11 @@ public interface GridQueryIndexing {
     public String schema(String cacheName);
 
     /**
-     * Check if passed statement is insert statemtn.
+     * Check if passed statement is insert statement eligible for streaming, throw an {@link IgniteSQLException} if not.
      *
      * @param nativeStmt Native statement.
-     * @return {@code True} if insert.
      */
-    public boolean isInsertStatement(PreparedStatement nativeStmt);
+    public void checkStatementStreamable(PreparedStatement nativeStmt);
 
     /**
      * Return row cache cleaner.


[43/50] [abbrv] ignite git commit: IGNITE:7862: flatten plugin updated to version 1.0.1

Posted by sb...@apache.org.
IGNITE:7862: flatten plugin updated to version 1.0.1


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1981dc66
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1981dc66
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1981dc66

Branch: refs/heads/ignite-zk
Commit: 1981dc66d9f3c6622c79d05b05cf97e203ccb416
Parents: 78e8627
Author: Peter Ivanov <mr...@gmail.com>
Authored: Fri Mar 16 15:07:04 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Fri Mar 16 15:09:06 2018 +0300

----------------------------------------------------------------------
 modules/platforms/dotnet/build.ps1 | 6 +++++-
 parent/pom.xml                     | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/1981dc66/modules/platforms/dotnet/build.ps1
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/build.ps1 b/modules/platforms/dotnet/build.ps1
index 70ce25d..59918b5 100644
--- a/modules/platforms/dotnet/build.ps1
+++ b/modules/platforms/dotnet/build.ps1
@@ -121,6 +121,10 @@ if (!$skipJava) {
         echo "Maven detected at $mv."
     }
 
+    # Install Maven Wrapper
+    cmd /c "$mv -N io.takari:maven:wrapper -Dmaven=3.5.2"
+    $mv = "mvnw.cmd"
+
     # Run Maven
     echo "Starting Java (Maven) build..."
     
@@ -256,4 +260,4 @@ if (!$skipNuGet) {
         }
 
     echo "NuGet packages created in '$pwd\$nupkgDir'."
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/1981dc66/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index 5d85f0d..d60f6c2 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -612,7 +612,7 @@
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>flatten-maven-plugin</artifactId>
-                <version>1.0.0-beta-3</version>
+                <version>1.0.1</version>
 
                 <configuration>
                     <flattenMode>minimum</flattenMode>


[03/50] [abbrv] ignite git commit: IGNITE-7566 .NET: Build scripts: stop build when one of the steps fails

Posted by sb...@apache.org.
IGNITE-7566 .NET: Build scripts: stop build when one of the steps fails


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0936f162
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0936f162
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0936f162

Branch: refs/heads/ignite-zk
Commit: 0936f162b84d0ff701ac020405e7e44f55fae52e
Parents: 8c3731e
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Wed Mar 7 11:10:08 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Wed Mar 7 11:10:08 2018 +0300

----------------------------------------------------------------------
 modules/platforms/dotnet/build-mono.sh | 5 +++++
 modules/platforms/dotnet/build.sh      | 5 +++++
 2 files changed, 10 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/0936f162/modules/platforms/dotnet/build-mono.sh
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/build-mono.sh b/modules/platforms/dotnet/build-mono.sh
old mode 100644
new mode 100755
index 6f73eb7..a0a01c1
--- a/modules/platforms/dotnet/build-mono.sh
+++ b/modules/platforms/dotnet/build-mono.sh
@@ -20,10 +20,15 @@
 # Builds Java and .NET with Mono
 #
 
+# Fail on error.
+set -e
+
+# Build Java.
 pushd .
 cd ../../..
 mvn clean package -DskipTests -Dmaven.javadoc.skip=true -Plgpl,-examples,-clean-libs,-release,-scala,-clientDocs
 popd
 
+# Build .NET.
 nuget restore Apache.Ignite.sln
 msbuild Apache.Ignite.sln /p:RunCodeAnalysis=false

http://git-wip-us.apache.org/repos/asf/ignite/blob/0936f162/modules/platforms/dotnet/build.sh
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/build.sh b/modules/platforms/dotnet/build.sh
index fc5b344..1a58ff0 100755
--- a/modules/platforms/dotnet/build.sh
+++ b/modules/platforms/dotnet/build.sh
@@ -20,10 +20,15 @@
 # Builds Java and .NET with .NET Core
 #
 
+# Fail on error.
+#set -e
+
+# Build Java.
 pushd .
 cd ../../..
 mvn clean package -DskipTests -Dmaven.javadoc.skip=true -Plgpl,-examples,-clean-libs,-release,-scala,-clientDocs
 popd
 
+# Build .NET
 cd Apache.Ignite.Core.Tests.DotNetCore
 dotnet build


[22/50] [abbrv] ignite git commit: IGNITE-7751 Use throttling to protect from checkpoint buffer overflow - Fixes #3611.

Posted by sb...@apache.org.
IGNITE-7751 Use throttling to protect from checkpoint buffer overflow - Fixes #3611.

Signed-off-by: Alexey Goncharuk <al...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/05d58bb6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/05d58bb6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/05d58bb6

Branch: refs/heads/ignite-zk
Commit: 05d58bb6efe35bfdd8bdb3e74dfd2627f6df7f9c
Parents: 8da7c9e
Author: Ivan Rakov <iv...@gmail.com>
Authored: Tue Mar 13 19:49:12 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Mar 13 19:49:12 2018 +0300

----------------------------------------------------------------------
 .../internal/pagemem/PageIdAllocator.java       |   4 +-
 .../pagemem/impl/PageMemoryNoStoreImpl.java     |   2 +-
 .../GridCacheDatabaseSharedManager.java         |  39 ++--
 .../persistence/pagemem/PageMemoryImpl.java     | 169 +++++++--------
 .../pagemem/PagesWriteSpeedBasedThrottle.java   |  16 +-
 .../persistence/pagemem/PagesWriteThrottle.java |  42 ++--
 .../pagemem/BPlusTreePageMemoryImplTest.java    |   2 +-
 .../BPlusTreeReuseListPageMemoryImplTest.java   |   3 +-
 ...gnitePageMemReplaceDelayedWriteUnitTest.java |   5 +-
 .../pagemem/IgniteThrottlingUnitTest.java       |   1 -
 .../pagemem/IndexStoragePageMemoryImplTest.java |   3 +-
 .../pagemem/PageMemoryImplNoLoadTest.java       |   3 +-
 .../persistence/pagemem/PageMemoryImplTest.java | 211 ++++++++++++++++++-
 13 files changed, 363 insertions(+), 137 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
index ae7da14..c6aeabe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
@@ -38,11 +38,11 @@ public interface PageIdAllocator {
     /**
      * Allocates a page from the space for the given partition ID and the given flags.
      *
-     * @param cacheId Cache Group ID.
+     * @param grpId Cache Group ID.
      * @param partId Partition ID.
      * @return Allocated page ID.
      */
-    public long allocatePage(int cacheId, int partId, byte flags) throws IgniteCheckedException;
+    public long allocatePage(int grpId, int partId, byte flags) throws IgniteCheckedException;
 
     /**
      * The given page is free now.

http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
index af1555e..7424af6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
@@ -254,7 +254,7 @@ public class PageMemoryNoStoreImpl implements PageMemory {
     }
 
     /** {@inheritDoc} */
-    @Override public long allocatePage(int cacheId, int partId, byte flags) {
+    @Override public long allocatePage(int grpId, int partId, byte flags) {
         memMetrics.incrementTotalAllocatedPages();
 
         long relPtr = borrowFreePage();

http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index d6a8a30..997f89a 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -193,6 +193,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     private final int walRebalanceThreshold = IgniteSystemProperties.getInteger(
         IGNITE_PDS_WAL_REBALANCE_THRESHOLD, 500_000);
 
+    /** Value of property for throttling policy override. */
+    private final String throttlingPolicyOverride = IgniteSystemProperties.getString(
+        IgniteSystemProperties.IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED);
+
     /** Checkpoint lock hold count. */
     private static final ThreadLocal<Integer> CHECKPOINT_LOCK_HOLD_COUNT = new ThreadLocal<Integer>() {
         @Override protected Integer initialValue() {
@@ -932,19 +936,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
             chpBufSize = cacheSize;
         }
 
-        PageMemoryImpl.ThrottlingPolicy plc = persistenceCfg.isWriteThrottlingEnabled()
-            ? PageMemoryImpl.ThrottlingPolicy.SPEED_BASED
-            : PageMemoryImpl.ThrottlingPolicy.NONE;
-
-        String val = IgniteSystemProperties.getString(IgniteSystemProperties.IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED);
-
-        if (val != null) {
-            if ("ratio".equalsIgnoreCase(val))
-                plc = PageMemoryImpl.ThrottlingPolicy.TARGET_RATIO_BASED;
-            else if ("speed".equalsIgnoreCase(val) || Boolean.valueOf(val))
-                plc = PageMemoryImpl.ThrottlingPolicy.SPEED_BASED;
-        }
-
         GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker;
 
         if (trackable)
@@ -985,7 +976,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
             changeTracker,
             this,
             memMetrics,
-            plc,
+            resolveThrottlingPolicy(),
             this
         );
 
@@ -994,6 +985,26 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
         return pageMem;
     }
 
+    /**
+     * Resolves throttling policy according to the settings.
+     */
+    @NotNull private PageMemoryImpl.ThrottlingPolicy resolveThrottlingPolicy() {
+        PageMemoryImpl.ThrottlingPolicy plc = persistenceCfg.isWriteThrottlingEnabled()
+            ? PageMemoryImpl.ThrottlingPolicy.SPEED_BASED
+            : PageMemoryImpl.ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY;
+
+        if (throttlingPolicyOverride != null) {
+            try {
+                plc = PageMemoryImpl.ThrottlingPolicy.valueOf(throttlingPolicyOverride.toUpperCase());
+            }
+            catch (IllegalArgumentException e) {
+                log.error("Incorrect value of IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED property: " +
+                    throttlingPolicyOverride + ". Default throttling policy " + plc + " will be used.");
+            }
+        }
+        return plc;
+    }
+
     /** {@inheritDoc} */
     @Override protected void checkRegionEvictionProperties(DataRegionConfiguration regCfg, DataStorageConfiguration dbCfg)
         throws IgniteCheckedException {

http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index fa10a1f..9f979f5 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -80,6 +80,7 @@ import org.apache.ignite.internal.util.lang.GridInClosure3X;
 import org.apache.ignite.internal.util.offheap.GridOffHeapOutOfMemoryException;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import static java.lang.Boolean.FALSE;
@@ -147,7 +148,7 @@ public class PageMemoryImpl implements PageMemoryEx {
     /** Page ID offset */
     public static final int PAGE_ID_OFFSET = 16;
 
-    /** Page cache ID offset. */
+    /** Page cache group ID offset. */
     public static final int PAGE_CACHE_ID_OFFSET = 24;
 
     /** Page pin counter offset. */
@@ -163,7 +164,7 @@ public class PageMemoryImpl implements PageMemoryEx {
      * 8b Marker/timestamp
      * 8b Relative pointer
      * 8b Page ID
-     * 4b Cache ID
+     * 4b Cache group ID
      * 4b Pin count
      * 8b Lock
      * 8b Temporary buffer
@@ -176,6 +177,10 @@ public class PageMemoryImpl implements PageMemoryEx {
     /** Tracking io. */
     private static final TrackingPageIO trackingIO = TrackingPageIO.VERSIONS.latest();
 
+    /** Checkpoint pool overflow error message. */
+    public static final String CHECKPOINT_POOL_OVERFLOW_ERROR_MSG = "Failed to allocate temporary buffer for checkpoint " +
+        "(increase checkpointPageBufferSize configuration property)";
+
     /** Page size. */
     private final int sysPageSize;
 
@@ -277,7 +282,7 @@ public class PageMemoryImpl implements PageMemoryEx {
         CheckpointLockStateChecker stateChecker,
         DataRegionMetricsImpl memMetrics,
         @Nullable ThrottlingPolicy throttlingPlc,
-        @Nullable CheckpointWriteProgressSupplier cpProgressProvider
+        @NotNull CheckpointWriteProgressSupplier cpProgressProvider
     ) {
         assert ctx != null;
         assert pageSize > 0;
@@ -294,7 +299,7 @@ public class PageMemoryImpl implements PageMemoryEx {
                 null;
         this.changeTracker = changeTracker;
         this.stateChecker = stateChecker;
-        this.throttlingPlc = throttlingPlc != null ? throttlingPlc : ThrottlingPolicy.NONE;
+        this.throttlingPlc = throttlingPlc != null ? throttlingPlc : ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY;
         this.cpProgressProvider = cpProgressProvider;
 
         storeMgr = ctx.pageStore();
@@ -363,22 +368,15 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /**
-     *
+     * Resolves instance of {@link PagesWriteThrottlePolicy} according to chosen throttle policy.
      */
     private void initWriteThrottle() {
-        if (!isThrottlingEnabled())
-            return;
-
-        if (cpProgressProvider == null) {
-            log.error("Write throttle can't start. CP progress provider not presented");
-
-            throttlingPlc = ThrottlingPolicy.NONE;
-        }
-
         if (throttlingPlc == ThrottlingPolicy.SPEED_BASED)
             writeThrottle = new PagesWriteSpeedBasedThrottle(this, cpProgressProvider, stateChecker, log);
-        else if(throttlingPlc == ThrottlingPolicy.TARGET_RATIO_BASED)
-            writeThrottle = new PagesWriteThrottle(this, cpProgressProvider, stateChecker);
+        else if (throttlingPlc == ThrottlingPolicy.TARGET_RATIO_BASED)
+            writeThrottle = new PagesWriteThrottle(this, cpProgressProvider, stateChecker, false);
+        else if (throttlingPlc == ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY)
+            writeThrottle = new PagesWriteThrottle(this, null, stateChecker, true);
     }
 
     /** {@inheritDoc} */
@@ -398,8 +396,8 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /** {@inheritDoc} */
-    @Override public void releasePage(int cacheId, long pageId, long page) {
-        Segment seg = segment(cacheId, pageId);
+    @Override public void releasePage(int grpId, long pageId, long page) {
+        Segment seg = segment(grpId, pageId);
 
         seg.readLock().lock();
 
@@ -412,18 +410,18 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /** {@inheritDoc} */
-    @Override public long readLock(int cacheId, long pageId, long page) {
-        return readLockPage(page, new FullPageId(pageId, cacheId), false);
+    @Override public long readLock(int grpId, long pageId, long page) {
+        return readLockPage(page, new FullPageId(pageId, grpId), false);
     }
 
     /** {@inheritDoc} */
-    @Override public void readUnlock(int cacheId, long pageId, long page) {
+    @Override public void readUnlock(int grpId, long pageId, long page) {
         readUnlockPage(page);
     }
 
     /** {@inheritDoc} */
-    @Override public long writeLock(int cacheId, long pageId, long page) {
-        return writeLock(cacheId, pageId, page, false);
+    @Override public long writeLock(int grpId, long pageId, long page) {
+        return writeLock(grpId, pageId, page, false);
     }
 
     /** {@inheritDoc} */
@@ -432,14 +430,14 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /** {@inheritDoc} */
-    @Override public long tryWriteLock(int cacheId, long pageId, long page) {
-        return tryWriteLockPage(page, new FullPageId(pageId, cacheId), true);
+    @Override public long tryWriteLock(int grpId, long pageId, long page) {
+        return tryWriteLockPage(page, new FullPageId(pageId, grpId), true);
     }
 
     /** {@inheritDoc} */
-    @Override public void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc,
+    @Override public void writeUnlock(int grpId, long pageId, long page, Boolean walPlc,
         boolean dirtyFlag) {
-        writeUnlock(cacheId, pageId, page, walPlc, dirtyFlag, false);
+        writeUnlock(grpId, pageId, page, walPlc, dirtyFlag, false);
     }
 
     /** {@inheritDoc} */
@@ -449,12 +447,12 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean isDirty(int cacheId, long pageId, long page) {
+    @Override public boolean isDirty(int grpId, long pageId, long page) {
         return isDirty(page);
     }
 
     /** {@inheritDoc} */
-    @Override public long allocatePage(int cacheId, int partId, byte flags) throws IgniteCheckedException {
+    @Override public long allocatePage(int grpId, int partId, byte flags) throws IgniteCheckedException {
         assert flags == PageIdAllocator.FLAG_DATA && partId <= PageIdAllocator.MAX_PARTITION_ID ||
             flags == PageIdAllocator.FLAG_IDX && partId == PageIdAllocator.INDEX_PARTITION :
             "flags = " + flags + ", partId = " + partId;
@@ -464,19 +462,19 @@ public class PageMemoryImpl implements PageMemoryEx {
         if (isThrottlingEnabled())
             writeThrottle.onMarkDirty(false);
 
-        long pageId = storeMgr.allocatePage(cacheId, partId, flags);
+        long pageId = storeMgr.allocatePage(grpId, partId, flags);
 
         assert PageIdUtils.pageIndex(pageId) > 0; //it's crucial for tracking pages (zero page is super one)
 
         // We need to allocate page in memory for marking it dirty to save it in the next checkpoint.
         // Otherwise it is possible that on file will be empty page which will be saved at snapshot and read with error
         // because there is no crc inside them.
-        Segment seg = segment(cacheId, pageId);
+        Segment seg = segment(grpId, pageId);
 
         DelayedDirtyPageWrite delayedWriter = delayedPageReplacementTracker != null
             ? delayedPageReplacementTracker.delayedPageWrite() : null;
 
-        FullPageId fullId = new FullPageId(pageId, cacheId);
+        FullPageId fullId = new FullPageId(pageId, grpId);
 
         seg.writeLock().lock();
 
@@ -484,15 +482,15 @@ public class PageMemoryImpl implements PageMemoryEx {
 
         try {
             long relPtr = seg.loadedPages.get(
-                cacheId,
+                grpId,
                 PageIdUtils.effectivePageId(pageId),
-                seg.partGeneration(cacheId, partId),
+                seg.partGeneration(grpId, partId),
                 INVALID_REL_PTR,
                 OUTDATED_REL_PTR
             );
 
             if (relPtr == OUTDATED_REL_PTR)
-                relPtr = refreshOutdatedPage(seg, cacheId, pageId, false);
+                relPtr = refreshOutdatedPage(seg, grpId, pageId, false);
 
             if (relPtr == INVALID_REL_PTR)
                 relPtr = seg.borrowOrAllocateFreePage(pageId);
@@ -528,7 +526,7 @@ public class PageMemoryImpl implements PageMemoryEx {
                         if (!ctx.wal().isAlwaysWriteFullPages())
                             ctx.wal().log(
                                 new InitNewPageRecord(
-                                    cacheId,
+                                    grpId,
                                     pageId,
                                     trackingIO.getType(),
                                     trackingIO.getVersion(), pageId
@@ -539,7 +537,7 @@ public class PageMemoryImpl implements PageMemoryEx {
                 }
             }
 
-            seg.loadedPages.put(cacheId, PageIdUtils.effectivePageId(pageId), relPtr, seg.partGeneration(cacheId, partId));
+            seg.loadedPages.put(grpId, PageIdUtils.effectivePageId(pageId), relPtr, seg.partGeneration(grpId, partId));
         }
         catch (IgniteOutOfMemoryException oom) {
             DataRegionConfiguration dataRegionCfg = getDataRegionConfiguration();
@@ -562,7 +560,7 @@ public class PageMemoryImpl implements PageMemoryEx {
         }
 
         //we have allocated 'tracking' page, we need to allocate regular one
-        return isTrackingPage ? allocatePage(cacheId, partId, flags) : pageId;
+        return isTrackingPage ? allocatePage(grpId, partId, flags) : pageId;
     }
 
     /**
@@ -596,7 +594,7 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean freePage(int cacheId, long pageId) throws IgniteCheckedException {
+    @Override public boolean freePage(int grpId, long pageId) throws IgniteCheckedException {
         assert false : "Free page should be never called directly when persistence is enabled.";
 
         return false;
@@ -613,25 +611,25 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /** {@inheritDoc} */
-    @Override public long acquirePage(int cacheId, long pageId) throws IgniteCheckedException {
-        return acquirePage(cacheId, pageId, false);
+    @Override public long acquirePage(int grpId, long pageId) throws IgniteCheckedException {
+        return acquirePage(grpId, pageId, false);
     }
 
     /** {@inheritDoc} */
-    @Override public long acquirePage(int cacheId, long pageId, boolean restore) throws IgniteCheckedException {
-        FullPageId fullId = new FullPageId(pageId, cacheId);
+    @Override public long acquirePage(int grpId, long pageId, boolean restore) throws IgniteCheckedException {
+        FullPageId fullId = new FullPageId(pageId, grpId);
 
         int partId = PageIdUtils.partId(pageId);
 
-        Segment seg = segment(cacheId, pageId);
+        Segment seg = segment(grpId, pageId);
 
         seg.readLock().lock();
 
         try {
             long relPtr = seg.loadedPages.get(
-                cacheId,
+                grpId,
                 PageIdUtils.effectivePageId(pageId),
-                seg.partGeneration(cacheId, partId),
+                seg.partGeneration(grpId, partId),
                 INVALID_REL_PTR,
                 INVALID_REL_PTR
             );
@@ -660,9 +658,9 @@ public class PageMemoryImpl implements PageMemoryEx {
         try {
             // Double-check.
             long relPtr = seg.loadedPages.get(
-                cacheId,
+                grpId,
                 PageIdUtils.effectivePageId(pageId),
-                seg.partGeneration(cacheId, partId),
+                seg.partGeneration(grpId, partId),
                 INVALID_REL_PTR,
                 OUTDATED_REL_PTR
             );
@@ -688,10 +686,10 @@ public class PageMemoryImpl implements PageMemoryEx {
                 setDirty(fullId, absPtr, false, false);
 
                 seg.loadedPages.put(
-                    cacheId,
+                    grpId,
                     PageIdUtils.effectivePageId(pageId),
                     relPtr,
-                    seg.partGeneration(cacheId, partId)
+                    seg.partGeneration(grpId, partId)
                 );
 
                 long pageAddr = absPtr + PAGE_OVERHEAD;
@@ -722,7 +720,7 @@ public class PageMemoryImpl implements PageMemoryEx {
             else if (relPtr == OUTDATED_REL_PTR) {
                 assert PageIdUtils.pageIndex(pageId) == 0 : fullId;
 
-                relPtr = refreshOutdatedPage(seg, cacheId, pageId, false);
+                relPtr = refreshOutdatedPage(seg, grpId, pageId, false);
 
                 absPtr = seg.absolute(relPtr);
 
@@ -764,7 +762,7 @@ public class PageMemoryImpl implements PageMemoryEx {
                 ByteBuffer buf = wrapPointer(pageAddr, pageSize());
 
                 try {
-                    storeMgr.read(cacheId, pageId, buf);
+                    storeMgr.read(grpId, pageId, buf);
                 }
                 catch (IgniteDataIntegrityViolationException ignore) {
                     U.warn(log, "Failed to read page (data integrity violation encountered, will try to " +
@@ -783,17 +781,17 @@ public class PageMemoryImpl implements PageMemoryEx {
 
     /**
      * @param seg Segment.
-     * @param cacheId Cache ID.
+     * @param grpId Cache group ID.
      * @param pageId Page ID.
      * @param rmv {@code True} if page should be removed.
      * @return Relative pointer to refreshed page.
      */
-    private long refreshOutdatedPage(Segment seg, int cacheId, long pageId, boolean rmv) {
+    private long refreshOutdatedPage(Segment seg, int grpId, long pageId, boolean rmv) {
         assert seg.writeLock().isHeldByCurrentThread();
 
-        int tag = seg.partGeneration(cacheId, PageIdUtils.partId(pageId));
+        int tag = seg.partGeneration(grpId, PageIdUtils.partId(pageId));
 
-        long relPtr = seg.loadedPages.refresh(cacheId, PageIdUtils.effectivePageId(pageId), tag);
+        long relPtr = seg.loadedPages.refresh(grpId, PageIdUtils.effectivePageId(pageId), tag);
 
         long absPtr = seg.absolute(relPtr);
 
@@ -815,13 +813,13 @@ public class PageMemoryImpl implements PageMemoryEx {
         }
 
         if (rmv)
-            seg.loadedPages.remove(cacheId, PageIdUtils.effectivePageId(pageId));
+            seg.loadedPages.remove(grpId, PageIdUtils.effectivePageId(pageId));
 
         if (seg.segCheckpointPages != null)
-            seg.segCheckpointPages.remove(new FullPageId(pageId, cacheId));
+            seg.segCheckpointPages.remove(new FullPageId(pageId, grpId));
 
         if (seg.dirtyPages != null)
-            seg.dirtyPages.remove(new FullPageId(pageId, cacheId));
+            seg.dirtyPages.remove(new FullPageId(pageId, grpId));
 
         return relPtr;
     }
@@ -991,7 +989,7 @@ public class PageMemoryImpl implements PageMemoryEx {
 
         memMetrics.resetDirtyPages();
 
-        if (isThrottlingEnabled())
+        if (throttlingPlc != ThrottlingPolicy.DISABLED)
             writeThrottle.onBeginCheckpoint();
 
         return new GridMultiCollectionWrapper<>(collections);
@@ -1001,7 +999,7 @@ public class PageMemoryImpl implements PageMemoryEx {
      * @return {@code True} if throttling is enabled.
      */
     private boolean isThrottlingEnabled() {
-        return throttlingPlc != ThrottlingPolicy.NONE;
+        return throttlingPlc != ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY && throttlingPlc != ThrottlingPolicy.DISABLED;
     }
 
     /** {@inheritDoc} */
@@ -1013,7 +1011,7 @@ public class PageMemoryImpl implements PageMemoryEx {
         for (Segment seg : segments)
             seg.segCheckpointPages = null;
 
-        if (isThrottlingEnabled())
+        if (throttlingPlc != ThrottlingPolicy.DISABLED)
             writeThrottle.onFinishCheckpoint();
     }
 
@@ -1340,8 +1338,8 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /** {@inheritDoc} */
-    @Override public long readLockForce(int cacheId, long pageId, long page) {
-        return readLockPage(page, new FullPageId(pageId, cacheId), true);
+    @Override public long readLockForce(int grpId, long pageId, long page) {
+        return readLockPage(page, new FullPageId(pageId, grpId), true);
     }
 
     /**
@@ -1400,9 +1398,7 @@ public class PageMemoryImpl implements PageMemoryEx {
             if (tmpRelPtr == INVALID_REL_PTR) {
                 rwLock.writeUnlock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
 
-                throw new IgniteException(
-                    "Failed to allocate temporary buffer for checkpoint " +
-                        "(increase checkpointPageBufferSize configuration property)");
+                throw new IgniteException(CHECKPOINT_POOL_OVERFLOW_ERROR_MSG + ": " + memMetrics.getName());
             }
 
             // Pin the page until checkpoint is not finished.
@@ -1472,7 +1468,7 @@ public class PageMemoryImpl implements PageMemoryEx {
         try {
             rwLock.writeUnlock(page + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
 
-            if (isThrottlingEnabled() && !restore && markDirty && !wasDirty)
+            if (throttlingPlc != ThrottlingPolicy.DISABLED && !restore && markDirty && !wasDirty)
                 writeThrottle.onMarkDirty(isInCheckpoint(fullId));
         }
         catch (AssertionError ex) {
@@ -1847,7 +1843,7 @@ public class PageMemoryImpl implements PageMemoryEx {
         /** Initial partition generation. */
         private static final int INIT_PART_GENERATION = 1;
 
-        /** Maps partition (cacheId, partId) to its generation. Generation is 1-based incrementing partition counter. */
+        /** Maps partition (grpId, partId) to its generation. Generation is 1-based incrementing partition counter. */
         private final Map<GroupPartitionId, Integer> partGenerationMap = new HashMap<>();
 
         /** */
@@ -1882,7 +1878,7 @@ public class PageMemoryImpl implements PageMemoryEx {
 
             pool = new PagePool(idx, poolRegion, null);
 
-            maxDirtyPages = throttlingPlc != ThrottlingPolicy.NONE
+            maxDirtyPages = throttlingPlc != ThrottlingPolicy.DISABLED
                 ? pool.pages() * 3 / 4
                 : Math.min(pool.pages() * 2 / 3, cpPoolPages);
         }
@@ -2599,37 +2595,37 @@ public class PageMemoryImpl implements PageMemoryEx {
         }
 
         /**
-         * Reads cache ID from the page at the given absolute pointer.
+         * Reads cache group ID from the page at the given absolute pointer.
          *
          * @param absPtr Absolute memory pointer to the page header.
-         * @return Cache ID written to the page.
+         * @return Cache group ID written to the page.
          */
-        private static int readPageCacheId(final long absPtr) {
+        private static int readPageGroupId(final long absPtr) {
             return GridUnsafe.getInt(absPtr + PAGE_CACHE_ID_OFFSET);
         }
 
         /**
-         * Writes cache ID from the page at the given absolute pointer.
+         * Writes cache group ID from the page at the given absolute pointer.
          *
          * @param absPtr Absolute memory pointer to the page header.
-         * @param cacheId Cache ID to write.
+         * @param grpId Cache group ID to write.
          */
-        private static void pageCacheId(final long absPtr, final int cacheId) {
-            GridUnsafe.putInt(absPtr + PAGE_CACHE_ID_OFFSET, cacheId);
+        private static void pageGroupId(final long absPtr, final int grpId) {
+            GridUnsafe.putInt(absPtr + PAGE_CACHE_ID_OFFSET, grpId);
         }
 
         /**
-         * Reads page ID and cache ID from the page at the given absolute pointer.
+         * Reads page ID and cache group ID from the page at the given absolute pointer.
          *
          * @param absPtr Absolute memory pointer to the page header.
          * @return Full page ID written to the page.
          */
         private static FullPageId fullPageId(final long absPtr) {
-            return new FullPageId(readPageId(absPtr), readPageCacheId(absPtr));
+            return new FullPageId(readPageId(absPtr), readPageGroupId(absPtr));
         }
 
         /**
-         * Writes page ID and cache ID from the page at the given absolute pointer.
+         * Writes page ID and cache group ID from the page at the given absolute pointer.
          *
          * @param absPtr Absolute memory pointer to the page header.
          * @param fullPageId Full page ID to write.
@@ -2637,7 +2633,7 @@ public class PageMemoryImpl implements PageMemoryEx {
         private static void fullPageId(final long absPtr, final FullPageId fullPageId) {
             pageId(absPtr, fullPageId.pageId());
 
-            pageCacheId(absPtr, fullPageId.groupId());
+            pageGroupId(absPtr, fullPageId.groupId());
         }
     }
 
@@ -2734,8 +2730,13 @@ public class PageMemoryImpl implements PageMemoryEx {
      * Throttling enabled and its type enum.
      */
     public enum ThrottlingPolicy {
-        /** Not throttled. */NONE,
-        /** Target ratio based: CP progress is used as border. */ TARGET_RATIO_BASED,
-        /** Speed based. CP writting speed and estimated ideal speed are used as border */ SPEED_BASED
+        /** All ways of throttling are disabled. */
+        DISABLED,
+        /** Only exponential throttling is used to protect from CP buffer overflow. */
+        CHECKPOINT_BUFFER_ONLY,
+        /** Target ratio based: CP progress is used as border. */
+        TARGET_RATIO_BASED,
+        /** Speed based. CP writting speed and estimated ideal speed are used as border */
+        SPEED_BASED
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java
index aaf5471..68fa529 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java
@@ -196,21 +196,23 @@ public class PagesWriteSpeedBasedThrottle implements PagesWriteThrottlePolicy {
                         markDirtySpeed,
                         curCpWriteSpeed);
 
-                    level = ThrottleMode.LIMITED;
+                    level = throttleParkTimeNs == 0 ? ThrottleMode.NO : ThrottleMode.LIMITED;
                 }
             }
         }
 
-        if (level == ThrottleMode.NO) {
-            exponentialBackoffCntr.set(0);
-
-            throttleParkTimeNs = 0;
-        }
-        else if (level == ThrottleMode.EXPONENTIAL) {
+        if (level == ThrottleMode.EXPONENTIAL) {
             int exponent = exponentialBackoffCntr.getAndIncrement();
 
             throttleParkTimeNs = (long)(STARTING_THROTTLE_NANOS * Math.pow(BACKOFF_RATIO, exponent));
         }
+        else {
+            if (isPageInCheckpoint)
+                exponentialBackoffCntr.set(0);
+
+            if (level == ThrottleMode.NO)
+                throttleParkTimeNs = 0;
+        }
 
         if (throttleParkTimeNs > 0) {
             recurrentLogIfNeed();

http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
index 78e5344..166cdcd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
@@ -32,6 +32,9 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
     /** Database manager. */
     private final CheckpointWriteProgressSupplier cpProgress;
 
+    /** If true, throttle will only protect from checkpoint buffer overflow, not from dirty pages ratio cap excess. */
+    private final boolean throttleOnlyPagesInCheckpoint;
+
     /** Checkpoint lock state checker. */
     private CheckpointLockStateChecker stateChecker;
 
@@ -41,30 +44,36 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
     /** Backoff ratio. Each next park will be this times longer. */
     private static final double BACKOFF_RATIO = 1.05;
 
-    /** Exponential backoff counter. */
-    private final AtomicInteger exponentialBackoffCntr = new AtomicInteger(0);
+    /** Counter for dirty pages ratio throttling. */
+    private final AtomicInteger notInCheckpointBackoffCntr = new AtomicInteger(0);
+
+    /** Counter for checkpoint buffer usage ratio throttling (we need a separate one due to IGNITE-7751). */
+    private final AtomicInteger inCheckpointBackoffCntr = new AtomicInteger(0);
+
     /**
      * @param pageMemory Page memory.
      * @param cpProgress Database manager.
      * @param stateChecker checkpoint lock state checker.
+     * @param throttleOnlyPagesInCheckpoint If true, throttle will only protect from checkpoint buffer overflow.
      */
     public PagesWriteThrottle(PageMemoryImpl pageMemory,
         CheckpointWriteProgressSupplier cpProgress,
-        CheckpointLockStateChecker stateChecker) {
+        CheckpointLockStateChecker stateChecker,
+        boolean throttleOnlyPagesInCheckpoint
+    ) {
         this.pageMemory = pageMemory;
         this.cpProgress = cpProgress;
         this.stateChecker = stateChecker;
+        this.throttleOnlyPagesInCheckpoint = throttleOnlyPagesInCheckpoint;
+
+        if (!throttleOnlyPagesInCheckpoint)
+            assert cpProgress != null : "cpProgress must be not null if ratio based throttling mode is used";
     }
 
     /** {@inheritDoc} */
     @Override public void onMarkDirty(boolean isPageInCheckpoint) {
         assert stateChecker.checkpointLockIsHeldByThread();
 
-        AtomicInteger writtenPagesCntr = cpProgress.writtenPagesCounter();
-
-        if (writtenPagesCntr == null)
-            return; // Don't throttle if checkpoint is not running.
-
         boolean shouldThrottle = false;
 
         if (isPageInCheckpoint) {
@@ -73,7 +82,12 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
             shouldThrottle = pageMemory.checkpointBufferPagesCount() > checkpointBufLimit;
         }
 
-        if (!shouldThrottle) {
+        if (!shouldThrottle && !throttleOnlyPagesInCheckpoint) {
+            AtomicInteger writtenPagesCntr = cpProgress.writtenPagesCounter();
+
+            if (writtenPagesCntr == null)
+                return; // Don't throttle if checkpoint is not running.
+
             int cpWrittenPages = writtenPagesCntr.get();
 
             int cpTotalPages = cpProgress.currentCheckpointPagesCount();
@@ -92,13 +106,15 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
             }
         }
 
+        AtomicInteger cntr = isPageInCheckpoint ? inCheckpointBackoffCntr : notInCheckpointBackoffCntr;
+
         if (shouldThrottle) {
-            int throttleLevel = exponentialBackoffCntr.getAndIncrement();
+            int throttleLevel = cntr.getAndIncrement();
 
             LockSupport.parkNanos((long)(STARTING_THROTTLE_NANOS * Math.pow(BACKOFF_RATIO, throttleLevel)));
         }
         else
-            exponentialBackoffCntr.set(0);
+            cntr.set(0);
     }
 
     /** {@inheritDoc} */
@@ -107,6 +123,8 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
 
     /** {@inheritDoc} */
     @Override public void onFinishCheckpoint() {
-        exponentialBackoffCntr.set(0);
+        inCheckpointBackoffCntr.set(0);
+
+        notInCheckpointBackoffCntr.set(0);
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
index a305b7f..3737204 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
@@ -76,7 +76,7 @@ public class BPlusTreePageMemoryImplTest extends BPlusTreeSelfTest {
             },
             () -> true,
             new DataRegionMetricsImpl(new DataRegionConfiguration()),
-            PageMemoryImpl.ThrottlingPolicy.NONE,
+            PageMemoryImpl.ThrottlingPolicy.DISABLED,
             null
         );
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
index 1fc34c5..0c786ad 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
@@ -23,7 +23,6 @@ import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.database.BPlusTreeReuseSelfTest;
@@ -77,7 +76,7 @@ public class BPlusTreeReuseListPageMemoryImplTest extends BPlusTreeReuseSelfTest
             },
             () -> true,
             new DataRegionMetricsImpl(new DataRegionConfiguration()),
-            PageMemoryImpl.ThrottlingPolicy.NONE,
+            PageMemoryImpl.ThrottlingPolicy.DISABLED,
             null
         );
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java
index a87a61e..c6f42e1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java
@@ -223,9 +223,8 @@ public class IgnitePageMemReplaceDelayedWriteUnitTest {
 
         DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
 
-        PageMemoryImpl memory
-            = new PageMemoryImpl(provider, sizes, sctx, pageSize,
-            pageWriter, null, () -> true, memMetrics, null, null);
+        PageMemoryImpl memory = new PageMemoryImpl(provider, sizes, sctx, pageSize,
+            pageWriter, null, () -> true, memMetrics, PageMemoryImpl.ThrottlingPolicy.DISABLED, null);
 
         memory.start();
         return memory;

http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java
index 1cef087..f9ca7e4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java
@@ -23,7 +23,6 @@ import java.util.concurrent.locks.LockSupport;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointWriteProgressSupplier;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.logger.NullLogger;
 import org.junit.Test;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
index 4495dc1..9087b1c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
@@ -24,7 +24,6 @@ import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.database.IndexStorageSelfTest;
@@ -92,7 +91,7 @@ public class IndexStoragePageMemoryImplTest extends IndexStorageSelfTest {
             },
             () -> true,
             new DataRegionMetricsImpl(new DataRegionConfiguration()),
-            PageMemoryImpl.ThrottlingPolicy.NONE,
+            PageMemoryImpl.ThrottlingPolicy.DISABLED,
             null
         );
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
index 3c169be..34fd93b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
@@ -86,7 +86,8 @@ public class PageMemoryImplNoLoadTest extends PageMemoryNoLoadSelfTest {
                 }
             },
             new DataRegionMetricsImpl(new DataRegionConfiguration()),
-            PageMemoryImpl.ThrottlingPolicy.NONE, null
+            PageMemoryImpl.ThrottlingPolicy.DISABLED,
+            null
         );
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
index 8f0ef39..31af118 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
@@ -17,24 +17,40 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.pagemem;
 
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
 import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
 import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageUtils;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointWriteProgressSupplier;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor;
 import org.apache.ignite.internal.util.lang.GridInClosure3X;
 import org.apache.ignite.plugin.PluginProvider;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.GridTestKernalContext;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.logger.GridTestLog4jLogger;
+import org.mockito.Mockito;
+
+import static org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.CHECKPOINT_POOL_OVERFLOW_ERROR_MSG;
 
 /**
  *
@@ -46,11 +62,14 @@ public class PageMemoryImplTest extends GridCommonAbstractTest {
     /** Page size. */
     private static final int PAGE_SIZE = 1024;
 
+    /** Max memory size. */
+    private static final int MAX_SIZE = 128;
+
     /**
      * @throws Exception if failed.
      */
     public void testThatAllocationTooMuchPagesCauseToOOMException() throws Exception {
-        PageMemoryImpl memory = createPageMemory();
+        PageMemoryImpl memory = createPageMemory(PageMemoryImpl.ThrottlingPolicy.DISABLED);
 
         try {
             while (!Thread.currentThread().isInterrupted())
@@ -64,15 +83,186 @@ public class PageMemoryImplTest extends GridCommonAbstractTest {
     }
 
     /**
-     *
+     * @throws Exception If failed.
+     */
+    public void testCheckpointBufferOverusageDontCauseWriteLockLeak() throws Exception {
+        PageMemoryImpl memory = createPageMemory(PageMemoryImpl.ThrottlingPolicy.DISABLED);
+
+        List<FullPageId> pages = new ArrayList<>();
+
+        try {
+            while (!Thread.currentThread().isInterrupted()) {
+                long pageId = memory.allocatePage(1, PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX);
+
+                FullPageId fullPageId = new FullPageId(pageId, 1);
+
+                pages.add(fullPageId);
+
+                acquireAndReleaseWriteLock(memory, fullPageId); //to set page id, otherwise we would fail with assertion error
+            }
+        }
+        catch (IgniteOutOfMemoryException ignore) {
+            //Success
+        }
+
+        memory.beginCheckpoint();
+
+        final AtomicReference<FullPageId> lastPage = new AtomicReference<>();
+
+        try {
+            for (FullPageId fullPageId : pages) {
+                lastPage.set(fullPageId);
+
+                acquireAndReleaseWriteLock(memory, fullPageId);
+            }
+        }
+        catch (Exception ex) {
+            assertTrue(ex.getMessage().startsWith(CHECKPOINT_POOL_OVERFLOW_ERROR_MSG));
+        }
+
+        memory.finishCheckpoint();
+
+        GridTestUtils.runAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    acquireAndReleaseWriteLock(memory, lastPage.get()); //we should be able get lock again
+                }
+                catch (IgniteCheckedException e) {
+                    throw new AssertionError(e);
+                }
+            }
+        }).get(getTestTimeout());
+    }
+
+    /**
+     * Tests that checkpoint buffer won't be overflowed with enabled CHECKPOINT_BUFFER_ONLY throttling.
+     * @throws Exception If failed.
      */
-    private PageMemoryImpl createPageMemory() throws Exception {
+    public void testCheckpointBufferCantOverflowMixedLoad() throws Exception {
+        testCheckpointBufferCantOverflowWithThrottlingMixedLoad(PageMemoryImpl.ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY);
+    }
+
+    /**
+     * Tests that checkpoint buffer won't be overflowed with enabled SPEED_BASED throttling.
+     * @throws Exception If failed.
+     */
+    public void testCheckpointBufferCantOverflowMixedLoadSpeedBased() throws Exception {
+        testCheckpointBufferCantOverflowWithThrottlingMixedLoad(PageMemoryImpl.ThrottlingPolicy.SPEED_BASED);
+    }
+
+    /**
+     * Tests that checkpoint buffer won't be overflowed with enabled TARGET_RATIO_BASED throttling.
+     * @throws Exception If failed.
+     */
+    public void testCheckpointBufferCantOverflowMixedLoadRatioBased() throws Exception {
+        testCheckpointBufferCantOverflowWithThrottlingMixedLoad(PageMemoryImpl.ThrottlingPolicy.TARGET_RATIO_BASED);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void testCheckpointBufferCantOverflowWithThrottlingMixedLoad(PageMemoryImpl.ThrottlingPolicy plc) throws Exception {
+        PageMemoryImpl memory = createPageMemory(plc);
+
+        List<FullPageId> pages = new ArrayList<>();
+
+        for (int i = 0; i < (MAX_SIZE - 10) * MB / PAGE_SIZE / 2; i++) {
+            long pageId = memory.allocatePage(1, PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX);
+
+            FullPageId fullPageId = new FullPageId(pageId, 1);
+
+            pages.add(fullPageId);
+
+            acquireAndReleaseWriteLock(memory, fullPageId);
+        }
+
+        memory.beginCheckpoint();
+
+        CheckpointMetricsTracker mockTracker = Mockito.mock(CheckpointMetricsTracker.class);
+
+        for (FullPageId checkpointPage : pages)
+            memory.getForCheckpoint(checkpointPage, ByteBuffer.allocate(PAGE_SIZE), mockTracker);
+
+        memory.finishCheckpoint();
+
+        for (int i = (int)((MAX_SIZE - 10) * MB / PAGE_SIZE / 2); i < (MAX_SIZE - 20) * MB / PAGE_SIZE; i++) {
+            long pageId = memory.allocatePage(1, PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX);
+
+            FullPageId fullPageId = new FullPageId(pageId, 1);
+
+            pages.add(fullPageId);
+
+            acquireAndReleaseWriteLock(memory, fullPageId);
+        }
+
+        memory.beginCheckpoint();
+
+        Collections.shuffle(pages); // Mix pages in checkpoint with clean pages
+
+        AtomicBoolean stop = new AtomicBoolean(false);
+
+        try {
+            GridTestUtils.runAsync(new Runnable() {
+                @Override public void run() {
+                    for (FullPageId page : pages) {
+                        if (ThreadLocalRandom.current().nextDouble() < 0.5) // Mark dirty 50% of pages
+                            try {
+                                acquireAndReleaseWriteLock(memory, page);
+
+                                if (stop.get())
+                                    break;
+                            }
+                            catch (IgniteCheckedException e) {
+                                log.error("runAsync ended with exception", e);
+
+                                fail();
+                            }
+                    }
+                }
+            }).get(5_000);
+        }
+        catch (IgniteFutureTimeoutCheckedException ex) {
+            // Expected.
+        }
+        finally {
+            stop.set(true);
+        }
+
+        memory.finishCheckpoint();
+    }
+
+    /**
+     * @param memory Memory.
+     * @param fullPageId Full page id.
+     * @throws IgniteCheckedException If acquiring lock failed.
+     */
+    private void acquireAndReleaseWriteLock(PageMemoryImpl memory, FullPageId fullPageId) throws IgniteCheckedException {
+        long page = memory.acquirePage(1, fullPageId.pageId());
+
+        long address = memory.writeLock(1, fullPageId.pageId(), page);
+
+        PageIO.setPageId(address, fullPageId.pageId());
+
+        PageIO.setType(address, PageIO.T_BPLUS_META);
+
+        PageUtils.putShort(address, PageIO.VER_OFF, (short)1);
+
+        memory.writeUnlock(1, fullPageId.pageId(), page, Boolean.FALSE, true);
+
+        memory.releasePage(1, fullPageId.pageId(), page);
+    }
+
+    /**
+     * @param throttlingPlc Throttling Policy.
+     * @throws Exception If creating mock failed.
+     */
+    private PageMemoryImpl createPageMemory(PageMemoryImpl.ThrottlingPolicy throttlingPlc) throws Exception {
         long[] sizes = new long[5];
 
         for (int i = 0; i < sizes.length; i++)
-            sizes[i] = 1024 * MB / 4;
+            sizes[i] = MAX_SIZE * MB / 4;
 
-        sizes[4] = 10 * MB;
+        sizes[4] = 5 * MB;
 
         DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
 
@@ -101,6 +291,13 @@ public class PageMemoryImplTest extends GridCommonAbstractTest {
             null
         );
 
+        CheckpointWriteProgressSupplier noThrottle = Mockito.mock(CheckpointWriteProgressSupplier.class);
+
+        Mockito.when(noThrottle.currentCheckpointPagesCount()).thenReturn(1_000_000);
+        Mockito.when(noThrottle.evictedPagesCntr()).thenReturn(new AtomicInteger(0));
+        Mockito.when(noThrottle.syncedPagesCounter()).thenReturn(new AtomicInteger(1_000_000));
+        Mockito.when(noThrottle.writtenPagesCounter()).thenReturn(new AtomicInteger(1_000_000));
+
         PageMemoryImpl mem = new PageMemoryImpl(
             provider,
             sizes,
@@ -118,8 +315,8 @@ public class PageMemoryImplTest extends GridCommonAbstractTest {
                 }
             },
             new DataRegionMetricsImpl(igniteCfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration()),
-            PageMemoryImpl.ThrottlingPolicy.NONE,
-            null
+            throttlingPlc,
+            noThrottle
         );
 
         mem.start();


[44/50] [abbrv] ignite git commit: .NET: Fix IgniteUuid type id (broken by IGNITE-7756 changes)

Posted by sb...@apache.org.
.NET: Fix IgniteUuid type id (broken by IGNITE-7756 changes)


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/094f93b4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/094f93b4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/094f93b4

Branch: refs/heads/ignite-zk
Commit: 094f93b4160b9a0d41ad958bc4ec5a737188e44e
Parents: 1981dc6
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Fri Mar 16 16:02:40 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Fri Mar 16 16:02:40 2018 +0300

----------------------------------------------------------------------
 .../dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/094f93b4/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs
index 8067a39..fab236c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs
@@ -167,8 +167,8 @@ namespace Apache.Ignite.Core.Impl.Binary
         /** Type: object written with Java OptimizedMarshaller. */
         public const byte OptimizedMarshaller = 254;
 
-        /** Type: platform object proxy. */
-        public const int IgniteUuid = 2018070327;
+        /** Type: Ignite UUID. */
+        public const int IgniteUuid = 63;
 
         /** Type ids. */
         private static readonly Dictionary<Type, byte> TypeIds = new Dictionary<Type, byte>


[09/50] [abbrv] ignite git commit: IGNITE-7803 Fixed regression in tests.

Posted by sb...@apache.org.
IGNITE-7803 Fixed regression in tests.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/90c0af88
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/90c0af88
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/90c0af88

Branch: refs/heads/ignite-zk
Commit: 90c0af887508ebace570cd13e7cf6b4369a995dc
Parents: 6dbcf7e
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Thu Mar 8 00:17:20 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Mar 8 00:17:20 2018 +0700

----------------------------------------------------------------------
 .../client/impl/ClientCacheFlagsCodecTest.java  | 35 +++----
 .../internal/client/GridClientCacheFlag.java    | 48 ++++++++++
 .../impl/connection/GridClientConnection.java   | 18 +---
 .../connection/GridClientNioTcpConnection.java  |  6 +-
 .../handlers/cache/GridCacheCommandHandler.java | 96 +++++++++-----------
 .../cache/GridCacheCommandHandlerSelfTest.java  |  9 +-
 .../http/jetty/GridJettyRestHandler.java        |  4 +-
 7 files changed, 120 insertions(+), 96 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/clients/src/test/java/org/apache/ignite/internal/client/impl/ClientCacheFlagsCodecTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/impl/ClientCacheFlagsCodecTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/impl/ClientCacheFlagsCodecTest.java
index 22e9a25..a9bda0a 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/client/impl/ClientCacheFlagsCodecTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/impl/ClientCacheFlagsCodecTest.java
@@ -18,16 +18,14 @@
 package org.apache.ignite.internal.client.impl;
 
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.EnumSet;
+import java.util.Set;
+
 import junit.framework.TestCase;
+
 import org.apache.ignite.internal.client.GridClientCacheFlag;
-import org.apache.ignite.internal.client.impl.connection.GridClientConnection;
-import org.apache.ignite.internal.processors.rest.handlers.cache.GridCacheCommandHandler;
 import org.apache.ignite.internal.util.typedef.F;
 
-import static org.apache.ignite.internal.client.GridClientCacheFlag.KEEP_BINARIES;
-
 /**
  * Tests conversions between GridClientCacheFlag.
  */
@@ -37,16 +35,13 @@ public class ClientCacheFlagsCodecTest extends TestCase {
      */
     public void testEncodingDecodingFullness() {
         for (GridClientCacheFlag f : GridClientCacheFlag.values()) {
-            if (f == KEEP_BINARIES)
-                continue;
-
-            int bits = GridClientConnection.encodeCacheFlags(Collections.singleton(f));
+            int bits = GridClientCacheFlag.encodeCacheFlags(EnumSet.of(f));
 
             assertTrue(bits != 0);
 
-            boolean out = GridCacheCommandHandler.parseCacheFlags(bits);
+            Set<GridClientCacheFlag> out = GridClientCacheFlag.parseCacheFlags(bits);
 
-            assertEquals(out, true);
+            assertTrue(out.contains(f));
         }
     }
 
@@ -54,9 +49,10 @@ public class ClientCacheFlagsCodecTest extends TestCase {
      * Tests that groups of client flags can be correctly converted to corresponding server flag groups.
      */
     public void testGroupEncodingDecoding() {
-        // all
+        // All.
         doTestGroup(GridClientCacheFlag.values());
-        // none
+
+        // None.
         doTestGroup();
     }
 
@@ -64,15 +60,14 @@ public class ClientCacheFlagsCodecTest extends TestCase {
      * @param flags Client flags to be encoded, decoded and checked.
      */
     private void doTestGroup(GridClientCacheFlag... flags) {
-        EnumSet<GridClientCacheFlag> flagSet = F.isEmpty(flags) ? EnumSet.noneOf(GridClientCacheFlag.class) :
-            EnumSet.copyOf(Arrays.asList(flags));
-
-        int bits = GridClientConnection.encodeCacheFlags(flagSet);
+        EnumSet<GridClientCacheFlag> flagSet = F.isEmpty(flags)
+            ? EnumSet.noneOf(GridClientCacheFlag.class)
+            : EnumSet.copyOf(Arrays.asList(flags));
 
-        boolean out = GridCacheCommandHandler.parseCacheFlags(bits);
+        int bits = GridClientCacheFlag.encodeCacheFlags(flagSet);
 
-        int length = flagSet.contains(KEEP_BINARIES) ? flagSet.size() - 1 : flagSet.size();
+        Set<GridClientCacheFlag> out = GridClientCacheFlag.parseCacheFlags(bits);
 
-        assertEquals(length > 0, out);
+        assertTrue(out.containsAll(flagSet));
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/core/src/main/java/org/apache/ignite/internal/client/GridClientCacheFlag.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/GridClientCacheFlag.java b/modules/core/src/main/java/org/apache/ignite/internal/client/GridClientCacheFlag.java
index 2e4831e..2e26dd8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/GridClientCacheFlag.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/GridClientCacheFlag.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.internal.client;
 
 import java.util.Collection;
+import java.util.EnumSet;
+import java.util.Set;
 
 /**
  * Cache projection flags that specify projection behaviour.
@@ -38,6 +40,12 @@ public enum GridClientCacheFlag {
     KEEP_BINARIES;
 
     /** */
+    public static final int SKIP_STORE_MASK = 0b1;
+
+    /** */
+    public static final int KEEP_BINARIES_MASK = 0b10;
+
+    /** */
     private static final GridClientCacheFlag[] VALS = values();
 
     /**
@@ -49,4 +57,44 @@ public enum GridClientCacheFlag {
     public static GridClientCacheFlag fromOrdinal(int ord) {
         return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
     }
+
+    /**
+     * Encodes cache flags to bit map.
+     *
+     * @param flagSet Set of flags to be encoded.
+     * @return Bit map.
+     */
+    public static int encodeCacheFlags(Collection<GridClientCacheFlag> flagSet) {
+        int bits = 0;
+
+        if (flagSet.contains(SKIP_STORE))
+            bits |= SKIP_STORE_MASK;
+
+        if (flagSet.contains(KEEP_BINARIES))
+            bits |= KEEP_BINARIES_MASK;
+
+        return bits;
+    }
+
+    /**
+     * Retrieves cache flags from corresponding bits.
+     *
+     * @param cacheFlagsBits Integer representation of cache flags bit set.
+     * @return Cache flags.
+     */
+    public static Set<GridClientCacheFlag> parseCacheFlags(int cacheFlagsBits) {
+        boolean skipStore = (cacheFlagsBits & SKIP_STORE_MASK) != 0;
+        boolean keepBinaries = (cacheFlagsBits & KEEP_BINARIES_MASK) != 0;
+
+        if (skipStore & keepBinaries)
+            return EnumSet.of(SKIP_STORE, KEEP_BINARIES);
+
+        if (skipStore)
+            return EnumSet.of(SKIP_STORE);
+
+        if (keepBinaries)
+            return EnumSet.of(KEEP_BINARIES);
+
+        return EnumSet.noneOf(GridClientCacheFlag.class);
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnection.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnection.java
index 0d7b9e3..c75bd24 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnection.java
@@ -25,6 +25,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 import javax.net.ssl.SSLContext;
+
 import org.apache.ignite.internal.client.GridClientCacheFlag;
 import org.apache.ignite.internal.client.GridClientClosedException;
 import org.apache.ignite.internal.client.GridClientDataMetrics;
@@ -106,21 +107,6 @@ public abstract class GridClientConnection {
     }
 
     /**
-     * Encodes cache flags to bit map.
-     *
-     * @param flagSet Set of flags to be encoded.
-     * @return Bit map.
-     */
-    public static int encodeCacheFlags(Collection<GridClientCacheFlag> flagSet) {
-        int bits = 0;
-
-        if (flagSet.contains(GridClientCacheFlag.SKIP_STORE))
-            bits |= 1;
-
-        return bits;
-    }
-
-    /**
      * Puts key-value pair into cache.
      *
      * @param cacheName Cache name.
@@ -172,7 +158,7 @@ public abstract class GridClientConnection {
      * @param destNodeId Destination node ID.
      * @return Whether entry was actually removed.
      * @throws GridClientConnectionResetException In case of error.
-     * @throws org.apache.ignite.internal.client.GridClientClosedException If client was manually closed before request was sent over network.
+     * @throws GridClientClosedException If client was manually closed before request was sent over network.
      */
     public abstract <K> GridClientFutureAdapter<Boolean> cacheRemove(String cacheName, K key,
         Set<GridClientCacheFlag> flags, UUID destNodeId)

http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
index e117cc8..6e55df8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
@@ -38,6 +38,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import javax.net.ssl.SSLContext;
+
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.client.GridClientAuthenticationException;
@@ -83,6 +84,7 @@ import org.jetbrains.annotations.Nullable;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.apache.ignite.internal.client.GridClientCacheFlag.KEEP_BINARIES;
+import static org.apache.ignite.internal.client.GridClientCacheFlag.encodeCacheFlags;
 import static org.apache.ignite.internal.client.impl.connection.GridClientConnectionCloseReason.CONN_IDLE;
 import static org.apache.ignite.internal.client.impl.connection.GridClientConnectionCloseReason.FAILED;
 import static org.apache.ignite.internal.processors.rest.client.message.GridClientCacheRequest.GridCacheOperation.APPEND;
@@ -97,8 +99,8 @@ import static org.apache.ignite.internal.processors.rest.client.message.GridClie
 import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.MARSHALLER;
 
 /**
- * This class performs request to grid over tcp protocol. Serialization is performed with marshaller
- * provided.
+ * This class performs request to grid over TCP protocol.
+ * Serialization is performed with marshaller provided.
  */
 public class GridClientNioTcpConnection extends GridClientConnection {
     /** */

http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandler.java
index 0a545b4..b1f49ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandler.java
@@ -51,6 +51,7 @@ import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.client.GridClientCacheFlag;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.processors.cache.CacheConfigurationOverride;
 import org.apache.ignite.internal.processors.cache.CacheInvokeEntry;
@@ -83,6 +84,8 @@ import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CacheMode.REPLICATED;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
 import static org.apache.ignite.internal.GridClosureCallMode.BALANCE;
+import static org.apache.ignite.internal.client.GridClientCacheFlag.KEEP_BINARIES;
+import static org.apache.ignite.internal.client.GridClientCacheFlag.SKIP_STORE;
 import static org.apache.ignite.internal.processors.query.QueryUtils.TEMPLATE_PARTITIONED;
 import static org.apache.ignite.internal.processors.query.QueryUtils.TEMPLATE_REPLICATED;
 import static org.apache.ignite.internal.processors.rest.GridRestCommand.ATOMIC_DECREMENT;
@@ -182,22 +185,6 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
     }
 
     /**
-     * Retrieves cache flags from corresponding bits.
-     *
-     * @param cacheFlagsBits Integer representation of cache flags bit set.
-     * @return Skip store flag.
-     */
-    public static boolean parseCacheFlags(int cacheFlagsBits) {
-        if (cacheFlagsBits == 0)
-            return false;
-
-        if ((cacheFlagsBits & 1) != 0)
-            return true;
-
-        return false;
-    }
-
-    /**
      * Handles append and prepend commands.
      *
      * @param ctx Kernal context.
@@ -377,7 +364,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
 
         final Object key = req0.key();
 
-        final boolean skipStore = parseCacheFlags(req0.cacheFlags());
+        final Set<GridClientCacheFlag> cacheFlags = GridClientCacheFlag.parseCacheFlags(req0.cacheFlags());
 
         try {
             GridRestCommand cmd = req0.command();
@@ -447,70 +434,70 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
                 }
 
                 case CACHE_CONTAINS_KEYS: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new ContainsKeysCommand(getKeys(req0)));
 
                     break;
                 }
 
                 case CACHE_CONTAINS_KEY: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new ContainsKeyCommand(key));
 
                     break;
                 }
 
                 case CACHE_GET: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new GetCommand(key));
 
                     break;
                 }
 
                 case CACHE_GET_AND_PUT: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new GetAndPutCommand(key, getValue(req0)));
 
                     break;
                 }
 
                 case CACHE_GET_AND_REPLACE: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new GetAndReplaceCommand(key, getValue(req0)));
 
                     break;
                 }
 
                 case CACHE_GET_AND_PUT_IF_ABSENT: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new GetAndPutIfAbsentCommand(key, getValue(req0)));
 
                     break;
                 }
 
                 case CACHE_PUT_IF_ABSENT: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new PutIfAbsentCommand(key, ttl, getValue(req0)));
 
                     break;
                 }
 
                 case CACHE_GET_ALL: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new GetAllCommand(getKeys(req0)));
 
                     break;
                 }
 
                 case CACHE_PUT: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key, new
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key, new
                         PutCommand(key, ttl, getValue(req0)));
 
                     break;
                 }
 
                 case CACHE_ADD: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new AddCommand(key, ttl, getValue(req0)));
 
                     break;
@@ -533,35 +520,35 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
                     // HashMap wrapping for correct serialization
                     map = new HashMap<>(map);
 
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new PutAllCommand(map));
 
                     break;
                 }
 
                 case CACHE_REMOVE: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new RemoveCommand(key));
 
                     break;
                 }
 
                 case CACHE_REMOVE_VALUE: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new RemoveValueCommand(key, getValue(req0)));
 
                     break;
                 }
 
                 case CACHE_REPLACE_VALUE: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new ReplaceValueCommand(key, getValue(req0), req0.value2()));
 
                     break;
                 }
 
                 case CACHE_GET_AND_REMOVE: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new GetAndRemoveCommand(key));
 
                     break;
@@ -573,7 +560,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
                     // HashSet wrapping for correct serialization
                     Set<Object> keys = map == null ? null : new HashSet<>(map.keySet());
 
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new RemoveAllCommand(keys));
 
                     break;
@@ -589,7 +576,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
                     GridCompoundFuture compFut = new GridCompoundFuture();
 
                     for (Object cName : cacheNames)
-                        compFut.add(executeCommand(req.destinationId(), req.clientId(), (String)cName, skipStore, key,
+                        compFut.add(executeCommand(req.destinationId(), req.clientId(), (String)cName, cacheFlags, key,
                             new RemoveAllCommand(null)));
 
                     compFut.markInitialized();
@@ -623,7 +610,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
                     if (val == null)
                         throw new IgniteCheckedException(GridRestCommandHandlerAdapter.missingParameter("val"));
 
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new ReplaceCommand(key, ttl, val));
 
                     break;
@@ -633,21 +620,21 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
                     final Object val1 = req0.value();
                     final Object val2 = req0.value2();
 
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new CasCommand(val2, val1, key));
 
                     break;
                 }
 
                 case CACHE_APPEND: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new AppendCommand(key, req0));
 
                     break;
                 }
 
                 case CACHE_PREPEND: {
-                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+                    fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
                         new PrependCommand(key, req0));
 
                     break;
@@ -680,12 +667,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
 
             return fut;
         }
-        catch (IgniteException e) {
-            U.error(log, "Failed to execute cache command: " + req, e);
-
-            return new GridFinishedFuture<>(e);
-        }
-        catch (IgniteCheckedException e) {
+        catch (IgniteException | IgniteCheckedException e) {
             U.error(log, "Failed to execute cache command: " + req, e);
 
             return new GridFinishedFuture<>(e);
@@ -741,7 +723,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
      * @param destId Target node Id for the operation. If {@code null} - operation could be executed anywhere.
      * @param clientId Client ID.
      * @param cacheName Cache name.
-     * @param skipStore Skip store.
+     * @param cacheFlags Cache flags.
      * @param key Key to set affinity mapping in the response.
      * @param op Operation to perform.
      * @return Operation result in future.
@@ -751,7 +733,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
         @Nullable UUID destId,
         UUID clientId,
         final String cacheName,
-        final boolean skipStore,
+        final Set<GridClientCacheFlag> cacheFlags,
         final Object key,
         final CacheProjectionCommand op) throws IgniteCheckedException {
 
@@ -761,8 +743,10 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
         if (locExec) {
             IgniteInternalCache<?, ?> prj = localCache(cacheName)
                 .forSubjectId(clientId)
-                .setSkipStore(skipStore)
-                .keepBinary();
+                .setSkipStore(cacheFlags.contains(SKIP_STORE));
+
+            if (cacheFlags.contains(KEEP_BINARIES))
+                prj = prj.keepBinary();
 
             return op.apply((IgniteInternalCache<Object, Object>)prj, ctx).
                 chain(resultWrapper((IgniteInternalCache<Object, Object>)prj, key));
@@ -773,7 +757,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
             ctx.task().setThreadContext(TC_NO_FAILOVER, true);
 
             return ctx.closure().callAsync(BALANCE,
-                new FlaggedCacheOperationCallable(clientId, cacheName, skipStore, op, key),
+                new FlaggedCacheOperationCallable(clientId, cacheName, cacheFlags, op, key),
                 prj.nodes());
         }
     }
@@ -902,7 +886,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
         /** */
         private final String cacheName;
         /** */
-        private final boolean skipStore;
+        private final Set<GridClientCacheFlag> cacheFlags;
         /** */
         private final CacheProjectionCommand op;
         /** */
@@ -916,19 +900,19 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
         /**
          * @param clientId Client ID.
          * @param cacheName Cache name.
-         * @param skipStore Skip store.
+         * @param cacheFlags Cache flags.
          * @param op Operation.
          * @param key Key.
          */
         private FlaggedCacheOperationCallable(UUID clientId,
             String cacheName,
-            boolean skipStore,
+            Set<GridClientCacheFlag> cacheFlags,
             CacheProjectionCommand op,
             Object key
         ) {
             this.clientId = clientId;
             this.cacheName = cacheName;
-            this.skipStore = skipStore;
+            this.cacheFlags = cacheFlags;
             this.op = op;
             this.key = key;
         }
@@ -937,8 +921,10 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
         @Override public GridRestResponse call() throws Exception {
             IgniteInternalCache<?, ?> prj = cache(g, cacheName)
                 .forSubjectId(clientId)
-                .setSkipStore(skipStore)
-                .keepBinary();
+                .setSkipStore(cacheFlags.contains(SKIP_STORE));
+
+            if (cacheFlags.contains(KEEP_BINARIES))
+                prj = prj.keepBinary();
 
             // Need to apply both operation and response transformation remotely
             // as cache could be inaccessible on local node and

http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandlerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandlerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandlerSelfTest.java
index e34992f..c94ebd8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandlerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandlerSelfTest.java
@@ -277,10 +277,15 @@ public class GridCacheCommandHandlerSelfTest extends GridCommonAbstractTest {
 
                             return fut;
                         }
+
                         // Rewriting flagOn result to keep intercepting invocations after it.
-                        else if ("setSkipStore".equals(mtd.getName()))
+                        if ("setSkipStore".equals(mtd.getName()))
+                            return proxy;
+
+                        if ("forSubjectId".equals(mtd.getName()))
                             return proxy;
-                        else if ("forSubjectId".equals(mtd.getName()))
+
+                        if ("keepBinary".equals(mtd.getName()))
                             return proxy;
 
                         return mtd.invoke(cache, args);

http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
----------------------------------------------------------------------
diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
index 1d80eea..85398cc 100644
--- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
+++ b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
@@ -67,6 +67,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.internal.client.GridClientCacheFlag.KEEP_BINARIES_MASK;
 import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_CONTAINS_KEYS;
 import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_GET_ALL;
 import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_PUT_ALL;
@@ -625,7 +626,8 @@ public class GridJettyRestHandler extends AbstractHandler {
                 if (val1 != null)
                     restReq0.value(val1);
 
-                restReq0.cacheFlags(intValue("cacheFlags", params, 0));
+                // Cache operations via REST will use binary objects.
+                restReq0.cacheFlags(intValue("cacheFlags", params, KEEP_BINARIES_MASK));
                 restReq0.ttl(longValue("exp", params, null));
 
                 if (cmd == CACHE_GET_ALL || cmd == CACHE_PUT_ALL || cmd == CACHE_REMOVE_ALL ||


[27/50] [abbrv] ignite git commit: IGNITE-6860 Add logging and additional info while OptimizedMarshaller fails to serialize or deserialize object.

Posted by sb...@apache.org.
IGNITE-6860 Add logging and additional info while OptimizedMarshaller fails to serialize or deserialize object.

Signed-off-by: Andrey Gura <ag...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c533e2cc
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c533e2cc
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c533e2cc

Branch: refs/heads/ignite-zk
Commit: c533e2ccb8dcbb2a9596befd4c4eda65a2f2fa0d
Parents: 7366809
Author: Ivan Daschinskiy <iv...@gmail.com>
Authored: Wed Mar 14 12:55:00 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Wed Mar 14 12:55:31 2018 +0300

----------------------------------------------------------------------
 .../internal/binary/BinaryClassDescriptor.java  | 382 ++++++++++---------
 .../internal/binary/BinaryFieldAccessor.java    |  61 ++-
 .../optimized/OptimizedObjectInputStream.java   | 103 ++---
 .../optimized/OptimizedObjectOutputStream.java  |  94 +++--
 .../binary/BinaryObjectExceptionSelfTest.java   |  41 ++
 .../OptimizedObjectStreamSelfTest.java          |  94 ++++-
 6 files changed, 478 insertions(+), 297 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/c533e2cc/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
index 3e75982..bac5d40 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
@@ -530,283 +530,297 @@ public class BinaryClassDescriptor {
      * @throws BinaryObjectException In case of error.
      */
     void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
-        assert obj != null;
-        assert writer != null;
-        assert mode != BinaryWriteMode.OPTIMIZED : "OptimizedMarshaller should not be used here: " + cls.getName();
+        try {
+            assert obj != null;
+            assert writer != null;
+            assert mode != BinaryWriteMode.OPTIMIZED : "OptimizedMarshaller should not be used here: " + cls.getName();
 
-        writer.typeId(typeId);
+            writer.typeId(typeId);
 
-        switch (mode) {
-            case P_BYTE:
-            case BYTE:
-                writer.writeByteFieldPrimitive((byte) obj);
+            switch (mode) {
+                case P_BYTE:
+                case BYTE:
+                    writer.writeByteFieldPrimitive((byte)obj);
 
-                break;
+                    break;
 
-            case P_SHORT:
-            case SHORT:
-                writer.writeShortFieldPrimitive((short)obj);
+                case P_SHORT:
+                case SHORT:
+                    writer.writeShortFieldPrimitive((short)obj);
 
-                break;
+                    break;
 
-            case P_INT:
-            case INT:
-                writer.writeIntFieldPrimitive((int) obj);
+                case P_INT:
+                case INT:
+                    writer.writeIntFieldPrimitive((int)obj);
 
-                break;
+                    break;
 
-            case P_LONG:
-            case LONG:
-                writer.writeLongFieldPrimitive((long) obj);
+                case P_LONG:
+                case LONG:
+                    writer.writeLongFieldPrimitive((long)obj);
 
-                break;
+                    break;
 
-            case P_FLOAT:
-            case FLOAT:
-                writer.writeFloatFieldPrimitive((float) obj);
+                case P_FLOAT:
+                case FLOAT:
+                    writer.writeFloatFieldPrimitive((float)obj);
 
-                break;
+                    break;
 
-            case P_DOUBLE:
-            case DOUBLE:
-                writer.writeDoubleFieldPrimitive((double) obj);
+                case P_DOUBLE:
+                case DOUBLE:
+                    writer.writeDoubleFieldPrimitive((double)obj);
 
-                break;
+                    break;
 
-            case P_CHAR:
-            case CHAR:
-                writer.writeCharFieldPrimitive((char) obj);
+                case P_CHAR:
+                case CHAR:
+                    writer.writeCharFieldPrimitive((char)obj);
 
-                break;
+                    break;
 
-            case P_BOOLEAN:
-            case BOOLEAN:
-                writer.writeBooleanFieldPrimitive((boolean) obj);
+                case P_BOOLEAN:
+                case BOOLEAN:
+                    writer.writeBooleanFieldPrimitive((boolean)obj);
 
-                break;
+                    break;
 
-            case DECIMAL:
-                writer.doWriteDecimal((BigDecimal)obj);
+                case DECIMAL:
+                    writer.doWriteDecimal((BigDecimal)obj);
 
-                break;
+                    break;
 
-            case STRING:
-                writer.doWriteString((String)obj);
+                case STRING:
+                    writer.doWriteString((String)obj);
 
-                break;
+                    break;
 
-            case UUID:
-                writer.doWriteUuid((UUID)obj);
+                case UUID:
+                    writer.doWriteUuid((UUID)obj);
 
-                break;
+                    break;
 
-            case DATE:
-                writer.doWriteDate((Date)obj);
+                case DATE:
+                    writer.doWriteDate((Date)obj);
 
-                break;
+                    break;
 
-            case TIMESTAMP:
-                writer.doWriteTimestamp((Timestamp)obj);
+                case TIMESTAMP:
+                    writer.doWriteTimestamp((Timestamp)obj);
 
-                break;
+                    break;
 
-            case TIME:
-                writer.doWriteTime((Time)obj);
+                case TIME:
+                    writer.doWriteTime((Time)obj);
 
-                break;
+                    break;
 
-            case BYTE_ARR:
-                writer.doWriteByteArray((byte[])obj);
+                case BYTE_ARR:
+                    writer.doWriteByteArray((byte[])obj);
 
-                break;
+                    break;
 
-            case SHORT_ARR:
-                writer.doWriteShortArray((short[]) obj);
+                case SHORT_ARR:
+                    writer.doWriteShortArray((short[])obj);
 
-                break;
+                    break;
 
-            case INT_ARR:
-                writer.doWriteIntArray((int[]) obj);
+                case INT_ARR:
+                    writer.doWriteIntArray((int[])obj);
 
-                break;
+                    break;
 
-            case LONG_ARR:
-                writer.doWriteLongArray((long[]) obj);
+                case LONG_ARR:
+                    writer.doWriteLongArray((long[])obj);
 
-                break;
+                    break;
 
-            case FLOAT_ARR:
-                writer.doWriteFloatArray((float[]) obj);
+                case FLOAT_ARR:
+                    writer.doWriteFloatArray((float[])obj);
 
-                break;
+                    break;
 
-            case DOUBLE_ARR:
-                writer.doWriteDoubleArray((double[]) obj);
+                case DOUBLE_ARR:
+                    writer.doWriteDoubleArray((double[])obj);
 
-                break;
+                    break;
 
-            case CHAR_ARR:
-                writer.doWriteCharArray((char[]) obj);
+                case CHAR_ARR:
+                    writer.doWriteCharArray((char[])obj);
 
-                break;
+                    break;
 
-            case BOOLEAN_ARR:
-                writer.doWriteBooleanArray((boolean[]) obj);
+                case BOOLEAN_ARR:
+                    writer.doWriteBooleanArray((boolean[])obj);
 
-                break;
+                    break;
 
-            case DECIMAL_ARR:
-                writer.doWriteDecimalArray((BigDecimal[]) obj);
+                case DECIMAL_ARR:
+                    writer.doWriteDecimalArray((BigDecimal[])obj);
 
-                break;
+                    break;
 
-            case STRING_ARR:
-                writer.doWriteStringArray((String[]) obj);
+                case STRING_ARR:
+                    writer.doWriteStringArray((String[])obj);
 
-                break;
+                    break;
 
-            case UUID_ARR:
-                writer.doWriteUuidArray((UUID[]) obj);
+                case UUID_ARR:
+                    writer.doWriteUuidArray((UUID[])obj);
 
-                break;
+                    break;
 
-            case DATE_ARR:
-                writer.doWriteDateArray((Date[]) obj);
+                case DATE_ARR:
+                    writer.doWriteDateArray((Date[])obj);
 
-                break;
+                    break;
 
-            case TIMESTAMP_ARR:
-                writer.doWriteTimestampArray((Timestamp[]) obj);
+                case TIMESTAMP_ARR:
+                    writer.doWriteTimestampArray((Timestamp[])obj);
 
-                break;
+                    break;
 
-            case TIME_ARR:
-                writer.doWriteTimeArray((Time[]) obj);
+                case TIME_ARR:
+                    writer.doWriteTimeArray((Time[])obj);
 
-                break;
+                    break;
 
-            case OBJECT_ARR:
-                writer.doWriteObjectArray((Object[])obj);
+                case OBJECT_ARR:
+                    writer.doWriteObjectArray((Object[])obj);
 
-                break;
+                    break;
 
-            case COL:
-                writer.doWriteCollection((Collection<?>)obj);
+                case COL:
+                    writer.doWriteCollection((Collection<?>)obj);
 
-                break;
+                    break;
 
-            case MAP:
-                writer.doWriteMap((Map<?, ?>)obj);
+                case MAP:
+                    writer.doWriteMap((Map<?, ?>)obj);
 
-                break;
+                    break;
 
-            case ENUM:
-                writer.doWriteEnum((Enum<?>)obj);
+                case ENUM:
+                    writer.doWriteEnum((Enum<?>)obj);
 
-                break;
+                    break;
 
-            case BINARY_ENUM:
-                writer.doWriteBinaryEnum((BinaryEnumObjectImpl)obj);
+                case BINARY_ENUM:
+                    writer.doWriteBinaryEnum((BinaryEnumObjectImpl)obj);
 
-                break;
+                    break;
 
-            case ENUM_ARR:
-                writer.doWriteEnumArray((Object[])obj);
+                case ENUM_ARR:
+                    writer.doWriteEnumArray((Object[])obj);
 
-                break;
+                    break;
 
-            case CLASS:
-                writer.doWriteClass((Class)obj);
+                case CLASS:
+                    writer.doWriteClass((Class)obj);
 
-                break;
+                    break;
 
-            case PROXY:
-                writer.doWriteProxy((Proxy)obj, intfs);
+                case PROXY:
+                    writer.doWriteProxy((Proxy)obj, intfs);
 
-                break;
+                    break;
 
-            case BINARY_OBJ:
-                writer.doWriteBinaryObject((BinaryObjectImpl)obj);
+                case BINARY_OBJ:
+                    writer.doWriteBinaryObject((BinaryObjectImpl)obj);
 
-                break;
+                    break;
 
-            case BINARY:
-                if (preWrite(writer, obj)) {
-                    try {
-                        if (serializer != null)
-                            serializer.writeBinary(obj, writer);
-                        else
-                            ((Binarylizable)obj).writeBinary(writer);
+                case BINARY:
+                    if (preWrite(writer, obj)) {
+                        try {
+                            if (serializer != null)
+                                serializer.writeBinary(obj, writer);
+                            else
+                                ((Binarylizable)obj).writeBinary(writer);
 
-                        postWrite(writer);
+                            postWrite(writer);
 
-                        // Check whether we need to update metadata.
-                        // The reason for this check is described in https://issues.apache.org/jira/browse/IGNITE-7138.
-                        if (obj.getClass() != BinaryMetadata.class && obj.getClass() != BinaryTreeMap.class) {
-                            int schemaId = writer.schemaId();
+                            // Check whether we need to update metadata.
+                            // The reason for this check is described in https://issues.apache.org/jira/browse/IGNITE-7138.
+                            if (obj.getClass() != BinaryMetadata.class && obj.getClass() != BinaryTreeMap.class) {
+                                int schemaId = writer.schemaId();
 
-                            if (schemaReg.schema(schemaId) == null) {
-                                // This is new schema, let's update metadata.
-                                BinaryMetadataCollector collector =
-                                    new BinaryMetadataCollector(typeId, typeName, mapper);
+                                if (schemaReg.schema(schemaId) == null) {
+                                    // This is new schema, let's update metadata.
+                                    BinaryMetadataCollector collector =
+                                        new BinaryMetadataCollector(typeId, typeName, mapper);
 
-                                if (serializer != null)
-                                    serializer.writeBinary(obj, collector);
-                                else
-                                    ((Binarylizable)obj).writeBinary(collector);
+                                    if (serializer != null)
+                                        serializer.writeBinary(obj, collector);
+                                    else
+                                        ((Binarylizable)obj).writeBinary(collector);
 
-                                BinarySchema newSchema = collector.schema();
+                                    BinarySchema newSchema = collector.schema();
 
-                                BinaryMetadata meta = new BinaryMetadata(typeId, typeName, collector.meta(),
-                                    affKeyFieldName, Collections.singleton(newSchema), false, null);
+                                    BinaryMetadata meta = new BinaryMetadata(typeId, typeName, collector.meta(),
+                                        affKeyFieldName, Collections.singleton(newSchema), false, null);
 
-                                ctx.updateMetadata(typeId, meta);
+                                    ctx.updateMetadata(typeId, meta);
 
-                                schemaReg.addSchema(newSchema.schemaId(), newSchema);
+                                    schemaReg.addSchema(newSchema.schemaId(), newSchema);
+                                }
                             }
-                        }
 
-                        postWriteHashCode(writer, obj);
-                    }
-                    finally {
-                        writer.popSchema();
+                            postWriteHashCode(writer, obj);
+                        }
+                        finally {
+                            writer.popSchema();
+                        }
                     }
-                }
 
-                break;
+                    break;
 
-            case OBJECT:
-                if (userType && !stableSchemaPublished) {
-                    // Update meta before write object with new schema
-                    BinaryMetadata meta = new BinaryMetadata(typeId, typeName, stableFieldsMeta,
-                        affKeyFieldName, Collections.singleton(stableSchema), false, null);
+                case OBJECT:
+                    if (userType && !stableSchemaPublished) {
+                        // Update meta before write object with new schema
+                        BinaryMetadata meta = new BinaryMetadata(typeId, typeName, stableFieldsMeta,
+                            affKeyFieldName, Collections.singleton(stableSchema), false, null);
 
-                    ctx.updateMetadata(typeId, meta);
+                        ctx.updateMetadata(typeId, meta);
 
-                    schemaReg.addSchema(stableSchema.schemaId(), stableSchema);
+                        schemaReg.addSchema(stableSchema.schemaId(), stableSchema);
 
-                    stableSchemaPublished = true;
-                }
+                        stableSchemaPublished = true;
+                    }
 
-                if (preWrite(writer, obj)) {
-                    try {
-                        for (BinaryFieldAccessor info : fields)
-                            info.write(obj, writer);
+                    if (preWrite(writer, obj)) {
+                        try {
+                            for (BinaryFieldAccessor info : fields)
+                                info.write(obj, writer);
 
-                        writer.schemaId(stableSchema.schemaId());
+                            writer.schemaId(stableSchema.schemaId());
 
-                        postWrite(writer);
-                        postWriteHashCode(writer, obj);
-                    }
-                    finally {
-                        writer.popSchema();
+                            postWrite(writer);
+                            postWriteHashCode(writer, obj);
+                        }
+                        finally {
+                            writer.popSchema();
+                        }
                     }
-                }
 
-                break;
+                    break;
 
-            default:
-                assert false : "Invalid mode: " + mode;
+                default:
+                    assert false : "Invalid mode: " + mode;
+            }
+        }
+        catch (Exception e) {
+            String msg;
+
+            if (S.INCLUDE_SENSITIVE && !F.isEmpty(typeName))
+                msg = "Failed to serialize object [typeName=" + typeName + ']';
+            else
+                msg = "Failed to serialize object [typeId=" + typeId + ']';
+
+            U.error(ctx.log(), msg, e);
+
+            throw new BinaryObjectException(msg, e);
         }
     }
 
@@ -871,10 +885,16 @@ public class BinaryClassDescriptor {
             return res;
         }
         catch (Exception e) {
+            String msg;
+
             if (S.INCLUDE_SENSITIVE && !F.isEmpty(typeName))
-                throw new BinaryObjectException("Failed to deserialize object [typeName=" + typeName + ']', e);
+                msg = "Failed to deserialize object [typeName=" + typeName + ']';
             else
-                throw new BinaryObjectException("Failed to deserialize object [typeId=" + typeId + ']', e);
+                msg = "Failed to deserialize object [typeId=" + typeId + ']';
+
+            U.error(ctx.log(), msg, e);
+
+            throw new BinaryObjectException(msg, e);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/c533e2cc/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java
index 567bee1..3277403 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java
@@ -150,7 +150,26 @@ public abstract class BinaryFieldAccessor {
      * @param writer Writer.
      * @throws BinaryObjectException If failed.
      */
-    public abstract void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException;
+    public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+        try {
+            write0(obj, writer);
+        }
+        catch (Exception ex) {
+            if (S.INCLUDE_SENSITIVE && !F.isEmpty(name))
+                throw new BinaryObjectException("Failed to write field [name=" + name + ']', ex);
+            else
+                throw new BinaryObjectException("Failed to write field [id=" + id + ']', ex);
+        }
+    }
+
+    /**
+     * Write field.
+     *
+     * @param obj Object.
+     * @param writer Writer.
+     * @throws BinaryObjectException If failed.
+     */
+    protected abstract void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException;
 
     /**
      * Read field.
@@ -215,7 +234,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+        @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
             writer.writeFieldIdNoSchemaUpdate(id);
 
             byte val = GridUnsafe.getByteField(obj, offset);
@@ -224,7 +243,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+        @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
             byte val = reader.readByte(id);
 
             GridUnsafe.putByteField(obj, offset, val);
@@ -245,7 +264,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+        @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
             writer.writeFieldIdNoSchemaUpdate(id);
 
             boolean val = GridUnsafe.getBooleanField(obj, offset);
@@ -254,7 +273,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+        @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
             boolean val = reader.readBoolean(id);
 
             GridUnsafe.putBooleanField(obj, offset, val);
@@ -275,7 +294,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+        @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
             writer.writeFieldIdNoSchemaUpdate(id);
 
             short val = GridUnsafe.getShortField(obj, offset);
@@ -284,7 +303,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+        @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
             short val = reader.readShort(id);
 
             GridUnsafe.putShortField(obj, offset, val);
@@ -305,7 +324,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+        @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
             writer.writeFieldIdNoSchemaUpdate(id);
 
             char val = GridUnsafe.getCharField(obj, offset);
@@ -314,7 +333,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+        @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
             char val = reader.readChar(id);
 
             GridUnsafe.putCharField(obj, offset, val);
@@ -335,7 +354,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+        @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
             writer.writeFieldIdNoSchemaUpdate(id);
 
             int val = GridUnsafe.getIntField(obj, offset);
@@ -344,7 +363,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+        @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
             int val = reader.readInt(id);
 
             GridUnsafe.putIntField(obj, offset, val);
@@ -365,7 +384,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+        @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
             writer.writeFieldIdNoSchemaUpdate(id);
 
             long val = GridUnsafe.getLongField(obj, offset);
@@ -374,7 +393,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+        @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
             long val = reader.readLong(id);
 
             GridUnsafe.putLongField(obj, offset, val);
@@ -395,7 +414,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+        @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
             writer.writeFieldIdNoSchemaUpdate(id);
 
             float val = GridUnsafe.getFloatField(obj, offset);
@@ -404,7 +423,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+        @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
             float val = reader.readFloat(id);
 
             GridUnsafe.putFloatField(obj, offset, val);
@@ -425,7 +444,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+        @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
             writer.writeFieldIdNoSchemaUpdate(id);
 
             double val = GridUnsafe.getDoubleField(obj, offset);
@@ -434,7 +453,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+        @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
             double val = reader.readDouble(id);
 
             GridUnsafe.putDoubleField(obj, offset, val);
@@ -466,7 +485,7 @@ public abstract class BinaryFieldAccessor {
         }
 
         /** {@inheritDoc} */
-        @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+        @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
             assert obj != null;
             assert writer != null;
 
@@ -483,17 +502,17 @@ public abstract class BinaryFieldAccessor {
 
             switch (mode(val)) {
                 case BYTE:
-                    writer.writeByteField((Byte) val);
+                    writer.writeByteField((Byte)val);
 
                     break;
 
                 case SHORT:
-                    writer.writeShortField((Short) val);
+                    writer.writeShortField((Short)val);
 
                     break;
 
                 case INT:
-                    writer.writeIntField((Integer) val);
+                    writer.writeIntField((Integer)val);
 
                     break;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/c533e2cc/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java
index 0bd7cc0..17eba66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java
@@ -342,7 +342,13 @@ class OptimizedObjectInputStream extends ObjectInputStream {
 
                 curCls = desc.describedClass();
 
-                return desc.read(this);
+                try {
+                    return desc.read(this);
+                }
+                catch (IOException e){
+                    throw new IOException("Failed to deserialize object [typeName=" +
+                        desc.describedClass().getName() + ']', e);
+                }
 
             default:
                 SB msg = new SB("Unexpected error occurred during unmarshalling");
@@ -443,76 +449,81 @@ class OptimizedObjectInputStream extends ObjectInputStream {
         for (int i = 0; i < fieldOffs.size(); i++) {
             OptimizedClassDescriptor.FieldInfo t = fieldOffs.get(i);
 
-            switch ((t.type())) {
-                case BYTE:
-                    byte resByte = readByte();
+            try {
+                switch ((t.type())) {
+                    case BYTE:
+                        byte resByte = readByte();
 
-                    if (t.field() != null)
-                        setByte(obj, t.offset(), resByte);
+                        if (t.field() != null)
+                            setByte(obj, t.offset(), resByte);
 
-                    break;
+                        break;
 
-                case SHORT:
-                    short resShort = readShort();
+                    case SHORT:
+                        short resShort = readShort();
 
-                    if (t.field() != null)
-                        setShort(obj, t.offset(), resShort);
+                        if (t.field() != null)
+                            setShort(obj, t.offset(), resShort);
 
-                    break;
+                        break;
 
-                case INT:
-                    int resInt = readInt();
+                    case INT:
+                        int resInt = readInt();
 
-                    if (t.field() != null)
-                        setInt(obj, t.offset(), resInt);
+                        if (t.field() != null)
+                            setInt(obj, t.offset(), resInt);
 
-                    break;
+                        break;
 
-                case LONG:
-                    long resLong = readLong();
+                    case LONG:
+                        long resLong = readLong();
 
-                    if (t.field() != null)
-                        setLong(obj, t.offset(), resLong);
+                        if (t.field() != null)
+                            setLong(obj, t.offset(), resLong);
 
-                    break;
+                        break;
 
-                case FLOAT:
-                    float resFloat = readFloat();
+                    case FLOAT:
+                        float resFloat = readFloat();
 
-                    if (t.field() != null)
-                        setFloat(obj, t.offset(), resFloat);
+                        if (t.field() != null)
+                            setFloat(obj, t.offset(), resFloat);
 
-                    break;
+                        break;
 
-                case DOUBLE:
-                    double resDouble = readDouble();
+                    case DOUBLE:
+                        double resDouble = readDouble();
 
-                    if (t.field() != null)
-                        setDouble(obj, t.offset(), resDouble);
+                        if (t.field() != null)
+                            setDouble(obj, t.offset(), resDouble);
 
-                    break;
+                        break;
 
-                case CHAR:
-                    char resChar = readChar();
+                    case CHAR:
+                        char resChar = readChar();
 
-                    if (t.field() != null)
-                        setChar(obj, t.offset(), resChar);
+                        if (t.field() != null)
+                            setChar(obj, t.offset(), resChar);
 
-                    break;
+                        break;
 
-                case BOOLEAN:
-                    boolean resBoolean = readBoolean();
+                    case BOOLEAN:
+                        boolean resBoolean = readBoolean();
 
-                    if (t.field() != null)
-                        setBoolean(obj, t.offset(), resBoolean);
+                        if (t.field() != null)
+                            setBoolean(obj, t.offset(), resBoolean);
 
-                    break;
+                        break;
 
-                case OTHER:
-                    Object resObject = readObject();
+                    case OTHER:
+                        Object resObject = readObject();
 
-                    if (t.field() != null)
-                        setObject(obj, t.offset(), resObject);
+                        if (t.field() != null)
+                            setObject(obj, t.offset(), resObject);
+                }
+            }
+            catch (IOException e) {
+                throw new IOException("Failed to deserialize field [name=" + t.name() + ']', e);
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/c533e2cc/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java
index 8a323a7..66da2da 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java
@@ -230,12 +230,19 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
                         mapper);
                 }
 
-                if (handle >= 0) {
-                    writeByte(HANDLE);
-                    writeInt(handle);
+                try {
+                    if (handle >= 0) {
+                        writeByte(HANDLE);
+
+                        writeInt(handle);
+                    }
+                    else
+                        desc.write(this, obj);
+                }
+                catch (IOException e){
+                    throw new IOException("Failed to serialize object [typeName=" +
+                        desc.describedClass().getName() + ']', e);
                 }
-                else
-                    desc.write(this, obj);
             }
         }
     }
@@ -476,58 +483,63 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
         for (int i = 0; i < fields.size(); i++) {
             OptimizedClassDescriptor.FieldInfo t = fields.get(i);
 
-            switch (t.type()) {
-                case BYTE:
-                    if (t.field() != null)
-                        writeByte(getByte(obj, t.offset()));
+            try {
+                switch (t.type()) {
+                    case BYTE:
+                        if (t.field() != null)
+                            writeByte(getByte(obj, t.offset()));
 
-                    break;
+                        break;
 
-                case SHORT:
-                    if (t.field() != null)
-                        writeShort(getShort(obj, t.offset()));
+                    case SHORT:
+                        if (t.field() != null)
+                            writeShort(getShort(obj, t.offset()));
 
-                    break;
+                        break;
 
-                case INT:
-                    if (t.field() != null)
-                        writeInt(getInt(obj, t.offset()));
+                    case INT:
+                        if (t.field() != null)
+                            writeInt(getInt(obj, t.offset()));
 
-                    break;
+                        break;
 
-                case LONG:
-                    if (t.field() != null)
-                        writeLong(getLong(obj, t.offset()));
+                    case LONG:
+                        if (t.field() != null)
+                            writeLong(getLong(obj, t.offset()));
 
-                    break;
+                        break;
 
-                case FLOAT:
-                    if (t.field() != null)
-                        writeFloat(getFloat(obj, t.offset()));
+                    case FLOAT:
+                        if (t.field() != null)
+                            writeFloat(getFloat(obj, t.offset()));
 
-                    break;
+                        break;
 
-                case DOUBLE:
-                    if (t.field() != null)
-                        writeDouble(getDouble(obj, t.offset()));
+                    case DOUBLE:
+                        if (t.field() != null)
+                            writeDouble(getDouble(obj, t.offset()));
 
-                    break;
+                        break;
 
-                case CHAR:
-                    if (t.field() != null)
-                        writeChar(getChar(obj, t.offset()));
+                    case CHAR:
+                        if (t.field() != null)
+                            writeChar(getChar(obj, t.offset()));
 
-                    break;
+                        break;
 
-                case BOOLEAN:
-                    if (t.field() != null)
-                        writeBoolean(getBoolean(obj, t.offset()));
+                    case BOOLEAN:
+                        if (t.field() != null)
+                            writeBoolean(getBoolean(obj, t.offset()));
 
-                    break;
+                        break;
 
-                case OTHER:
-                    if (t.field() != null)
-                        writeObject0(getObject(obj, t.offset()));
+                    case OTHER:
+                        if (t.field() != null)
+                            writeObject0(getObject(obj, t.offset()));
+                }
+            }
+            catch (IOException e) {
+                throw new IOException("Failed to serialize field [name=" + t.name() + ']', e);
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/c533e2cc/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectExceptionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectExceptionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectExceptionSelfTest.java
index a3ee93a..0a10fe7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectExceptionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectExceptionSelfTest.java
@@ -24,6 +24,9 @@ import java.util.UUID;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.binary.BinaryBasicNameMapper;
 import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
 import org.apache.ignite.configuration.BinaryConfiguration;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -149,6 +152,44 @@ public class BinaryObjectExceptionSelfTest extends GridCommonAbstractTest {
         assertEquals("Fields count must match \"Unexpected field type\" exception count", fields.length, unexpectedCnt);
     }
 
+    /**
+     * Test verbose logging of object marshalling.
+     *
+     * @throws Exception If failed.
+     */
+    public void testFailedMarshallingLogging() throws Exception {
+        BinaryMarshaller marshaller = createStandaloneBinaryMarshaller();
+
+        try {
+            marshaller.marshal(new Value1());
+        }
+        catch (BinaryObjectException ex) {
+            assertTrue(ex.getMessage().contains(
+                "object [typeName=org.apache.ignite.internal.binary.BinaryObjectExceptionSelfTest$Value1"));
+
+            assertTrue(ex.getCause().getMessage().contains("field [name=objVal"));
+        }
+    }
+
+    /** */
+    private static class Value1{
+        /** */
+        Value2 objVal = new Value2();
+
+        /** */
+        static class Value2 implements Binarylizable {
+            /** */
+            @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+                throw new RuntimeException("bad object");
+            }
+
+            /** */
+            @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+                throw new RuntimeException("bad object");
+            }
+        }
+    }
+
     /** */
     private enum EnumValues {
         /** */

http://git-wip-us.apache.org/repos/asf/ignite/blob/c533e2cc/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectStreamSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectStreamSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectStreamSelfTest.java
index ed0862b..5efc5b9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectStreamSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectStreamSelfTest.java
@@ -56,6 +56,7 @@ import java.util.TreeSet;
 import java.util.UUID;
 import java.util.Vector;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.util.io.GridUnsafeDataInput;
@@ -67,7 +68,6 @@ import org.apache.ignite.marshaller.MarshallerExclusions;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jetbrains.annotations.Nullable;
-import java.util.concurrent.ConcurrentHashMap;
 
 import static org.junit.Assert.assertArrayEquals;
 
@@ -146,13 +146,9 @@ public class OptimizedObjectStreamSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testBoolean() throws Exception {
-        boolean val = true;
-
-        assertEquals(new Boolean(val), marshalUnmarshal(val));
-
-        val = false;
+        assertEquals(Boolean.TRUE, marshalUnmarshal(Boolean.TRUE));
 
-        assertEquals(new Boolean(val), marshalUnmarshal(val));
+        assertEquals(Boolean.FALSE, marshalUnmarshal(Boolean.FALSE));
     }
 
     /**
@@ -275,6 +271,49 @@ public class OptimizedObjectStreamSelfTest extends GridCommonAbstractTest {
     }
 
     /**
+     * Test informative exception message while failed object unmarshalling
+     *
+     * @throws Exception If failed.
+     */
+    public void testFailedUnmarshallingLogging() throws Exception {
+        OptimizedMarshaller marsh = new OptimizedMarshaller(true);
+
+        marsh.setContext(CTX);
+
+        try {
+            marsh.unmarshal(marsh.marshal(new BadDeserializableObject()), null);
+        }
+        catch (IgniteCheckedException ex) {
+            assertTrue(ex.getCause().getMessage().contains(
+                "object [typeName=org.apache.ignite.internal.marshaller.optimized.OptimizedObjectStreamSelfTest$BadDeserializableObject]"));
+
+            assertTrue(ex.getCause().getCause().getMessage().contains("field [name=val"));
+        }
+    }
+
+
+    /**
+     * Test informative exception message while failed object marshalling
+     *
+     * @throws Exception If failed.
+     */
+    public void testFailedMarshallingLogging() throws Exception {
+        OptimizedMarshaller marsh = new OptimizedMarshaller(true);
+
+        marsh.setContext(CTX);
+
+        try {
+            marsh.marshal(new BadSerializableObject());
+        }
+        catch (IgniteCheckedException ex) {
+            assertTrue(ex.getCause().getMessage().contains(
+                "object [typeName=org.apache.ignite.internal.marshaller.optimized.OptimizedObjectStreamSelfTest$BadSerializableObject]"));
+
+            assertTrue(ex.getCause().getCause().getMessage().contains("field [name=val"));
+        }
+    }
+
+    /**
      * @throws Exception If failed.
      */
     public void testPool() throws Exception {
@@ -1028,7 +1067,7 @@ public class OptimizedObjectStreamSelfTest extends GridCommonAbstractTest {
             marshalUnmarshal(new CustomWriteObjectMethodObject("test"));
         }
         catch (IOException e) {
-            assert e.getCause() instanceof NotActiveException;
+            assert e.getCause().getCause() instanceof NotActiveException;
         }
     }
 
@@ -2249,4 +2288,43 @@ public class OptimizedObjectStreamSelfTest extends GridCommonAbstractTest {
             return res;
         }
     }
+
+    /** */
+    static class BadDeserializableObject implements Serializable {
+        /** */
+        BadDeserializableValue val = new BadDeserializableValue();
+    }
+
+    /** */
+    static class BadDeserializableValue implements Serializable {
+        /** */
+        private void writeObject(ObjectOutputStream os) throws IOException{
+            os.write(10);
+        }
+
+        /** */
+        private void readObject(ObjectInputStream os){
+            throw new RuntimeException("bad object");
+        }
+    }
+
+    /** */
+    static class BadSerializableObject implements Serializable {
+        /** */
+        BadSerializableValue val = new BadSerializableValue();
+    }
+
+    /** */
+    static class BadSerializableValue implements Serializable {
+        /** */
+        private void writeObject(ObjectOutputStream os){
+            throw new RuntimeException("bad object");
+        }
+
+        /** */
+        private void readObject(ObjectInputStream os){
+            throw new RuntimeException("bad object");
+        }
+    }
+
 }
\ No newline at end of file


[42/50] [abbrv] ignite git commit: IGNITE-7879: Fixed splitter logic for DISTINCT with subqueries. This closes #3634.

Posted by sb...@apache.org.
IGNITE-7879: Fixed splitter logic for DISTINCT with subqueries. This closes #3634.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/78e8627a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/78e8627a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/78e8627a

Branch: refs/heads/ignite-zk
Commit: 78e8627adae904774aa8513adfbedd0b225cc86e
Parents: f623ef1
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Fri Mar 16 14:38:38 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Mar 16 14:38:38 2018 +0300

----------------------------------------------------------------------
 .../query/h2/sql/GridSqlQuerySplitter.java      |  3 +-
 .../query/SqlPushDownFunctionTest.java          | 78 ++++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite.java          |  2 +
 3 files changed, 82 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/78e8627a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index c96b486..d2fb149 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -962,7 +962,8 @@ public class GridSqlQuerySplitter {
                 select.setColumn(i, expr);
             }
 
-            if (isAllRelatedToTables(tblAliases, GridSqlQuerySplitter.<GridSqlAlias>newIdentityHashSet(), expr)) {
+            if (isAllRelatedToTables(tblAliases, GridSqlQuerySplitter.<GridSqlAlias>newIdentityHashSet(), expr)
+                && !hasAggregates(expr)) {
                 // Push down the whole expression.
                 pushDownColumn(tblAliases, cols, wrapAlias, expr, 0);
             }

http://git-wip-us.apache.org/repos/asf/ignite/blob/78e8627a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPushDownFunctionTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPushDownFunctionTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPushDownFunctionTest.java
new file mode 100644
index 0000000..9e78770
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPushDownFunctionTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import java.util.List;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for schemas.
+ */
+public class SqlPushDownFunctionTest extends GridCommonAbstractTest {
+    /** Node. */
+    private IgniteEx node;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        node = (IgniteEx)startGrid();
+
+        startGrid(2);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     */
+    public void testPushDownFunction() {
+        sql("CREATE TABLE Person(id INTEGER PRIMARY KEY, company_id INTEGER)");
+        sql("CREATE TABLE Company(id INTEGER PRIMARY KEY, name VARCHAR)");
+
+        sql("INSERT INTO Person(id,company_id) VALUES (1, 1), (2, 2), (3, 3)");
+        sql("INSERT INTO Company(id,name) VALUES (1,'n1'), (2,'n2'), (3,'n3')");
+
+        sql("SELECT p.id, sum(p.id) FROM person p " +
+            "LEFT JOIN (select distinct id from company) as c on c.id=p.company_id " +
+            "GROUP BY p.id");
+
+        sql("SELECT count(1) FROM person p " +
+            "LEFT JOIN (select id from company union select id from company) as c on c.id=p.company_id");
+
+        sql("SELECT count(1) FROM person p " +
+            "LEFT JOIN (select id from company union select id from company) as c on c.id=p.company_id");
+
+        sql("SELECT 1, (select count(1)) FROM person p " +
+            "LEFT JOIN (select id from company union select id from company) as c on c.id=p.company_id");
+    }
+
+    /**
+     * @param sql SQL query.
+     * @return Results.
+     */
+    private List<List<?>> sql(String sql) {
+        GridQueryProcessor qryProc = node.context().query();
+
+        SqlFieldsQuery qry = new SqlFieldsQuery(sql).setSchema("PUBLIC");
+
+        return qryProc.querySqlFields(qry, true).getAll();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/78e8627a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index c01eaa6..35d1be2 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -157,6 +157,7 @@ import org.apache.ignite.internal.processors.query.IgniteSqlSkipReducerOnUpdateD
 import org.apache.ignite.internal.processors.query.IgniteSqlSplitterSelfTest;
 import org.apache.ignite.internal.processors.query.LazyQuerySelfTest;
 import org.apache.ignite.internal.processors.query.MultipleStatementsSqlQuerySelfTest;
+import org.apache.ignite.internal.processors.query.SqlPushDownFunctionTest;
 import org.apache.ignite.internal.processors.query.SqlSchemaSelfTest;
 import org.apache.ignite.internal.processors.query.h2.GridH2IndexingInMemSelfTest;
 import org.apache.ignite.internal.processors.query.h2.GridH2IndexingOffheapSelfTest;
@@ -230,6 +231,7 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
         // Queries tests.
         suite.addTestSuite(LazyQuerySelfTest.class);
         suite.addTestSuite(IgniteSqlSplitterSelfTest.class);
+        suite.addTestSuite(SqlPushDownFunctionTest.class);
         suite.addTestSuite(IgniteSqlSegmentedIndexSelfTest.class);
         suite.addTestSuite(IgniteCachelessQueriesSelfTest.class);
         suite.addTestSuite(IgniteSqlSegmentedIndexMultiNodeSelfTest.class);


[12/50] [abbrv] ignite git commit: IGNITE-7876: Adopt SVM Linear Binary Classification Model and Trainer to the new Partitioned Dataset

Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/LocalLinearSVMBinaryClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/LocalLinearSVMBinaryClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/LocalLinearSVMBinaryClassificationTrainerTest.java
deleted file mode 100644
index a9cb54c..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/LocalLinearSVMBinaryClassificationTrainerTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm.binary;
-
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
-import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationTrainer;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link DenseLocalOnHeapMatrix}.
- */
-public class LocalLinearSVMBinaryClassificationTrainerTest extends GenericLinearSVMBinaryClassificationTrainerTest {
-    /** */
-    public LocalLinearSVMBinaryClassificationTrainerTest() {
-        super(
-            new SVMLinearBinaryClassificationTrainer()
-                .withLambda(0.2)
-                .withAmountOfIterations(10)
-                .withAmountOfLocIterations(20),
-            false,
-            1e-2);
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/DistributedLinearSVMMultiClassClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/DistributedLinearSVMMultiClassClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/DistributedLinearSVMMultiClassClassificationTrainerTest.java
deleted file mode 100644
index 6806e0b..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/DistributedLinearSVMMultiClassClassificationTrainerTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm.multi;
-
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
-import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationTrainer;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link DenseLocalOnHeapMatrix}.
- */
-public class DistributedLinearSVMMultiClassClassificationTrainerTest extends GenericLinearSVMMultiClassClassificationTrainerTest {
-    /** */
-    public DistributedLinearSVMMultiClassClassificationTrainerTest() {
-        super(
-            new SVMLinearMultiClassClassificationTrainer(),
-            true,
-            1e-2);
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/GenericLinearSVMMultiClassClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/GenericLinearSVMMultiClassClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/GenericLinearSVMMultiClassClassificationTrainerTest.java
deleted file mode 100644
index 8c6083d..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/GenericLinearSVMMultiClassClassificationTrainerTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm.multi;
-
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledDataset;
-import org.apache.ignite.ml.svm.BaseSVMTest;
-import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationModel;
-import org.junit.Test;
-
-/**
- * Base class for all linear regression trainers.
- */
-public class GenericLinearSVMMultiClassClassificationTrainerTest extends BaseSVMTest {
-    /** */
-    private final Trainer<SVMLinearMultiClassClassificationModel, LabeledDataset> trainer;
-
-    /** */
-    private boolean isDistributed;
-
-    /** */
-    private final double precision;
-
-    /** */
-    GenericLinearSVMMultiClassClassificationTrainerTest(
-        Trainer<SVMLinearMultiClassClassificationModel, LabeledDataset> trainer,
-        boolean isDistributed,
-        double precision) {
-        super();
-        this.trainer = trainer;
-        this.precision = precision;
-        this.isDistributed = isDistributed;
-    }
-
-    /**
-     * Test trainer on classification model y = x.
-     */
-    @Test
-    public void testTrainWithTheLinearlySeparableCase() {
-        if (isDistributed)
-            IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        double[][] mtx =
-            new double[][] {
-                {-10.0, 12.0},
-                {-5.0, 14.0},
-                {-3.0, 18.0},
-                {13.0, -1.0},
-                {10.0, -2.0},
-                {15.0, -3.0}};
-        double[] lbs = new double[] {1.0, 1.0, 1.0, 2.0, 2.0, 2.0};
-        LabeledDataset dataset = new LabeledDataset(mtx, lbs, null, isDistributed);
-
-
-        SVMLinearMultiClassClassificationModel mdl = trainer.train(dataset);
-        TestUtils.assertEquals(1.0, mdl.apply(new DenseLocalOnHeapVector(new double[] {-2.0, 15})), precision);
-        TestUtils.assertEquals(2.0, mdl.apply(new DenseLocalOnHeapVector(new double[] {12, -5})), precision);
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/LocalLinearSVMMultiClassClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/LocalLinearSVMMultiClassClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/LocalLinearSVMMultiClassClassificationTrainerTest.java
deleted file mode 100644
index a239c95..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/LocalLinearSVMMultiClassClassificationTrainerTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm.multi;
-
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
-import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationTrainer;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link DenseLocalOnHeapMatrix}.
- */
-public class LocalLinearSVMMultiClassClassificationTrainerTest extends GenericLinearSVMMultiClassClassificationTrainerTest {
-    /** */
-    public LocalLinearSVMMultiClassClassificationTrainerTest() {
-        super(
-            new SVMLinearMultiClassClassificationTrainer()
-                .withLambda(0.2)
-                .withAmountOfIterations(10)
-                .withAmountOfLocIterations(20),
-            false,
-            1e-2);
-    }
-}


[07/50] [abbrv] ignite git commit: IGNITE-7880 Web Console: Return enum values from SQL queries.

Posted by sb...@apache.org.
IGNITE-7880 Web Console: Return enum values from SQL queries.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a8170f78
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a8170f78
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a8170f78

Branch: refs/heads/ignite-zk
Commit: a8170f78a384af024bdd483294f2f241bef5a7f8
Parents: caba7cc
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Wed Mar 7 18:33:28 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Mar 7 18:33:28 2018 +0700

----------------------------------------------------------------------
 .../apache/ignite/internal/visor/query/VisorQueryUtils.java | 9 +++++++++
 1 file changed, 9 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/a8170f78/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java
index aa4cb48..a47acf6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java
@@ -195,6 +195,15 @@ public class VisorQueryUtils {
             }
 
             if (meta != null) {
+                if (meta.isEnum()) {
+                    try {
+                        return obj.deserialize().toString();
+                    }
+                    catch (BinaryObjectException ignore) {
+                        // NO-op.
+                    }
+                }
+
                 SB buf = new SB(meta.typeName());
 
                 if (meta.fieldNames() != null) {


[23/50] [abbrv] ignite git commit: IGNITE-7895 Web Console: Replaced PhantomJS with headless Chrome. Cleanup dev dependencies. Added docker image to run tests on TeamCity.

Posted by sb...@apache.org.
IGNITE-7895 Web Console: Replaced PhantomJS with headless Chrome. Cleanup dev dependencies. Added docker image to run tests on TeamCity.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/536e8254
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/536e8254
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/536e8254

Branch: refs/heads/ignite-zk
Commit: 536e825483deec864ee665d53cc8a695771c4aa0
Parents: 05d58bb
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Wed Mar 14 11:23:29 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Mar 14 11:23:29 2018 +0700

----------------------------------------------------------------------
 modules/web-console/frontend/package.json       | 19 ++------
 .../web-console/frontend/test/ci/.dockerignore  |  4 ++
 modules/web-console/frontend/test/ci/Dockerfile | 36 ++++++++++++++
 .../frontend/test/ci/docker-compose.yml         | 25 ++++++++++
 .../frontend/test/karma.conf.babel.js           | 12 +++--
 .../frontend/test/protractor.conf.js            | 50 --------------------
 6 files changed, 77 insertions(+), 69 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/536e8254/modules/web-console/frontend/package.json
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 100e644..fd301b2 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -8,8 +8,6 @@
     "dev": "npm start",
     "build": "webpack --config ./webpack/webpack.prod.babel.js",
     "test": "karma start ./test/karma.conf.js",
-    "enve2e": "node ./test/e2e/envtools.js start",
-    "teste2e": "node ./test/e2e/testcafe.js --env=true",
     "eslint": "eslint --format node_modules/eslint-friendly-formatter app/ controllers/ ignite_modules/ -- --eff-by-issue"
   },
   "author": "",
@@ -106,29 +104,18 @@
   },
   "devDependencies": {
     "chai": "4.1.0",
-    "copy-dir": "^0.3.0",
-    "fs-extra": "^4.0.2",
-    "glob": "^7.1.2",
+    "glob": "7.1.2",
     "jasmine-core": "2.6.4",
     "karma": "1.7.0",
     "karma-babel-preprocessor": "6.0.1",
+    "karma-chrome-launcher": "2.2.0",
     "karma-jasmine": "1.1.0",
     "karma-mocha": "1.3.0",
     "karma-mocha-reporter": "2.2.3",
-    "karma-phantomjs-launcher": "1.0.4",
     "karma-teamcity-reporter": "1.0.0",
     "karma-webpack": "2.0.4",
-    "minimist": "^1.2.0",
     "mocha": "3.4.2",
-    "mocha-teamcity-reporter": "1.1.1",
-    "mongodb": "2.2.33",
-    "node-cmd": "3.0.0",
-    "objectid": "3.2.1",
-    "phantomjs-prebuilt": "2.1.14",
     "sinon": "2.3.8",
-    "testcafe": "0.18.5",
-    "testcafe-angular-selectors": "0.3.0",
-    "type-detect": "4.0.3",
-    "util": "0.10.3"
+    "type-detect": "4.0.3"
   }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/536e8254/modules/web-console/frontend/test/ci/.dockerignore
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/test/ci/.dockerignore b/modules/web-console/frontend/test/ci/.dockerignore
new file mode 100644
index 0000000..bcac98f
--- /dev/null
+++ b/modules/web-console/frontend/test/ci/.dockerignore
@@ -0,0 +1,4 @@
+.git
+*Dockerfile*
+*docker-compose*
+*/node_modules*
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/536e8254/modules/web-console/frontend/test/ci/Dockerfile
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/test/ci/Dockerfile b/modules/web-console/frontend/test/ci/Dockerfile
new file mode 100644
index 0000000..a1b7c67
--- /dev/null
+++ b/modules/web-console/frontend/test/ci/Dockerfile
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM alpine:edge
+
+RUN apk --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/testing/ add \
+ nodejs nodejs-npm chromium xwininfo xvfb dbus eudev ttf-freefont fluxbox
+
+ENV CHROME_BIN /usr/bin/chromium-browser
+
+# Install frontend & backend apps.
+RUN mkdir -p /opt/web-console
+
+# Copy source.
+WORKDIR /opt/web-console
+COPY . ./frontend
+
+# Install node modules.
+WORKDIR /opt/web-console/frontend
+RUN npm install
+
+ENTRYPOINT ["npm", "test"]

http://git-wip-us.apache.org/repos/asf/ignite/blob/536e8254/modules/web-console/frontend/test/ci/docker-compose.yml
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/test/ci/docker-compose.yml b/modules/web-console/frontend/test/ci/docker-compose.yml
new file mode 100644
index 0000000..69667e6
--- /dev/null
+++ b/modules/web-console/frontend/test/ci/docker-compose.yml
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: '2'
+services:
+  unit_tests:
+    build:
+      context: '../../'
+      dockerfile: './test/ci/Dockerfile'
+    environment:
+      - TEST_REPORTER=teamcity
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/536e8254/modules/web-console/frontend/test/karma.conf.babel.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/test/karma.conf.babel.js b/modules/web-console/frontend/test/karma.conf.babel.js
index 30e168c..48104c8 100644
--- a/modules/web-console/frontend/test/karma.conf.babel.js
+++ b/modules/web-console/frontend/test/karma.conf.babel.js
@@ -35,7 +35,7 @@ export default (config) => {
         ],
 
         plugins: [
-            require('karma-phantomjs-launcher'),
+            require('karma-chrome-launcher'),
             require('karma-teamcity-reporter'),
             require('karma-mocha-reporter'),
             require('karma-webpack'),
@@ -57,7 +57,7 @@ export default (config) => {
         // Test results reporter to use
         // possible values: 'dots', 'progress'
         // available reporters: https://npmjs.org/browse/keyword/karma-reporter.
-        reporters: ['mocha'],
+        reporters: [process.env.TEST_REPORTER || 'mocha'],
 
         mochaReporter: {
             showDiff: true
@@ -78,7 +78,13 @@ export default (config) => {
 
         // start these browsers
         // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
-        browsers: ['PhantomJS'],
+        browsers: ['ChromeHeadlessNoSandbox'],
+        customLaunchers: {
+            ChromeHeadlessNoSandbox: {
+                base: 'ChromeHeadless',
+                flags: ['--no-sandbox']
+            }
+        },
 
         // Continuous Integration mode
         // if true, Karma captures browsers, runs the tests and exits

http://git-wip-us.apache.org/repos/asf/ignite/blob/536e8254/modules/web-console/frontend/test/protractor.conf.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/test/protractor.conf.js b/modules/web-console/frontend/test/protractor.conf.js
deleted file mode 100644
index 4ee1c69..0000000
--- a/modules/web-console/frontend/test/protractor.conf.js
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// exports.config = {
-//   specs: ['test/e2e/*.js'],
-//   capabilities: {
-
-//   }
-// };
-
-exports.config = {
-    seleniumAddress: 'http://localhost:4444/wd/hub',
-
-    capabilities: {
-        browserName: 'chrome'
-    // 'browserName': 'phantomjs',
-
-    // /*
-    //  * Can be used to specify the phantomjs binary path.
-    //  * This can generally be ommitted if you installed phantomjs globally.
-    //  */
-    // 'phantomjs.binary.path': require('phantomjs').path,
-
-    // /*
-    //  * Command line args to pass to ghostdriver, phantomjs's browser driver.
-    //  * See https://github.com/detro/ghostdriver#faq
-    //  */
-    // 'phantomjs.ghostdriver.cli.args': ['--loglevel=DEBUG']
-    },
-
-    specs: ['test/e2e/*.js'],
-
-    jasmineNodeOpts: {
-        showColors: true
-    }
-};


[35/50] [abbrv] ignite git commit: .NET: Suppress incorrect code inspection errors

Posted by sb...@apache.org.
.NET: Suppress incorrect code inspection errors


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9fd6f4a4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9fd6f4a4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9fd6f4a4

Branch: refs/heads/ignite-zk
Commit: 9fd6f4a4f64581f3ec4a5a3a7e489226afa1ed0b
Parents: 22eda40
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Thu Mar 15 16:48:46 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Thu Mar 15 16:48:46 2018 +0300

----------------------------------------------------------------------
 .../dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs | 1 +
 .../platforms/dotnet/Apache.Ignite.Linq/Impl/ExpressionWalker.cs    | 1 +
 2 files changed, 2 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/9fd6f4a4/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs
index cc12caa..1621183 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs
@@ -203,6 +203,7 @@ namespace Apache.Ignite.Core.Impl.Common
         /// Throws an exception if first argument is not null.
         /// </summary>
         // ReSharper disable once UnusedParameter.Local
+        // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Local
         private static void ThrowIfMultipleInterfaces(object check, Type userType, Type interfaceType)
         {
             if (check != null)

http://git-wip-us.apache.org/repos/asf/ignite/blob/9fd6f4a4/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/ExpressionWalker.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/ExpressionWalker.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/ExpressionWalker.cs
index 36655bc..0d6306a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/ExpressionWalker.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/ExpressionWalker.cs
@@ -119,6 +119,7 @@ namespace Apache.Ignite.Linq.Impl
         /// Tries to find QuerySourceReferenceExpression
         /// </summary>
         public static QuerySourceReferenceExpression GetQuerySourceReference(Expression expression,
+            // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Global
             bool throwWhenNotFound = true)
         {
             var reference = expression as QuerySourceReferenceExpression;