You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2018/03/19 06:26:38 UTC
[01/50] [abbrv] ignite git commit: IGNITE-7889: .NET: LINQ: Fix
GroupBy with Where
Repository: ignite
Updated Branches:
refs/heads/ignite-zk a65d955d8 -> 00917728b
IGNITE-7889: .NET: LINQ: Fix GroupBy with Where
This closes #3608
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/64d920bf
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/64d920bf
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/64d920bf
Branch: refs/heads/ignite-zk
Commit: 64d920bf59e18b383f7e2f230f8f4087406e38fe
Parents: a11bcf5
Author: Alexey Popov <ta...@gmail.com>
Authored: Tue Mar 6 20:47:18 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Tue Mar 6 20:47:18 2018 +0300
----------------------------------------------------------------------
.../Cache/Query/Linq/CacheLinqTest.Functions.cs | 48 ++++++++++++++++++++
.../Impl/CacheQueryModelVisitor.cs | 4 ++
2 files changed, 52 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64d920bf/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Functions.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Functions.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Functions.cs
index a672d5a..a6632d9 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Functions.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Functions.cs
@@ -110,6 +110,54 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
}
/// <summary>
+ /// Tests the GroupBy with Where subquery.
+ /// </summary>
+ [Test]
+ public void TestGroupBySubquery()
+ {
+ var persons = GetPersonCache().AsCacheQueryable().Where(p => p.Value.OrganizationId == 1000);
+ var orgs = GetOrgCache().AsCacheQueryable();
+
+ // Simple, unordered
+ CollectionAssert.AreEquivalent(new[] { 1000 },
+ persons.GroupBy(x => x.Value.OrganizationId).Select(x => x.Key).ToArray());
+
+ // Aggregate
+ Assert.AreEqual(1000,
+ persons.GroupBy(x => x.Value.OrganizationId).Select(x => x.Key).OrderBy(x => x).First());
+
+ // Ordering and count
+ var res1 =
+ from p in persons
+ orderby p.Value.Name
+ group p by p.Value.OrganizationId
+ into gs
+ orderby gs.Key
+ where gs.Count() > 10
+ select new { Count = gs.Count(), OrgId = gs.Key, AvgAge = gs.Average(x => x.Value.Age) };
+
+ var resArr = res1.ToArray();
+
+ Assert.AreEqual(new[]
+ {
+ new {Count = PersonCount/2, OrgId = 1000, AvgAge = (double) PersonCount/2 - 1},
+ }, resArr);
+
+ // Join and sum
+ var res2 = persons.Join(orgs.Where(o => o.Key > 10), p => p.Value.OrganizationId, o => o.Key,
+ (p, o) => new { p, o })
+ .GroupBy(x => x.o.Value.Name)
+ .Select(g => new { Org = g.Key, AgeSum = g.Select(x => x.p.Value.Age).Sum() });
+
+ var resArr2 = res2.ToArray();
+
+ Assert.AreEqual(new[]
+ {
+ new {Org = "Org_0", AgeSum = persons.Where(x => x.Value.OrganizationId == 1000).Sum(x => x.Value.Age)},
+ }, resArr2);
+ }
+
+ /// <summary>
/// Tests the union.
/// </summary>
[Test]
http://git-wip-us.apache.org/repos/asf/ignite/blob/64d920bf/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
index 56c3ff5..223eea6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
@@ -374,6 +374,10 @@ namespace Apache.Ignite.Linq.Impl
foreach (var join in subQuery.QueryModel.BodyClauses.OfType<JoinClause>())
VisitJoinClause(join, queryModel, i++);
+ i = 0;
+ foreach (var where in subQuery.QueryModel.BodyClauses.OfType<WhereClause>())
+ VisitWhereClause(where, i++, false);
+
// Append grouping
_builder.Append("group by (");
[11/50] [abbrv] ignite git commit: IGNITE-7881 Tests for using
TreeMap or TreeSet as cache key.
Posted by sb...@apache.org.
IGNITE-7881 Tests for using TreeMap or TreeSet as cache key.
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/29d56aed
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/29d56aed
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/29d56aed
Branch: refs/heads/ignite-zk
Commit: 29d56aedeae7c00ffe997530aeb7624d5863df5b
Parents: 4ee181c
Author: Ilya Kasnacheev <il...@gmail.com>
Authored: Mon Mar 12 15:19:10 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Mon Mar 12 15:19:10 2018 +0300
----------------------------------------------------------------------
.../internal/binary/BinaryTreeSelfTest.java | 273 +++++++++++++++----
1 file changed, 226 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/29d56aed/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java
index 9481340..819e766 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryTreeSelfTest.java
@@ -17,6 +17,11 @@
package org.apache.ignite.internal.binary;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import javax.cache.Cache.Entry;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.binary.BinaryObject;
@@ -29,11 +34,6 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.jetbrains.annotations.NotNull;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
/**
* Tests for TreeMap and TreeSet structures.
*/
@@ -71,8 +71,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
*
* @throws Exception If failed.
*/
- public void testTreeMapRegularNoComparator() throws Exception {
- checkTreeMap(false, false);
+ public void testTreeMapAsValueRegularNoComparator() throws Exception {
+ checkTreeMapAsValue(false, false);
}
/**
@@ -80,8 +80,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
*
* @throws Exception If failed.
*/
- public void testTreeMapRegularComparator() throws Exception {
- checkTreeMap(false, true);
+ public void testTreeMapAsValueRegularComparator() throws Exception {
+ checkTreeMapAsValue(false, true);
}
/**
@@ -89,8 +89,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
*
* @throws Exception If failed.
*/
- public void testTreeMapBinaryNoComparator() throws Exception {
- checkTreeMap(true, false);
+ public void testTreeMapAsValueBinaryNoComparator() throws Exception {
+ checkTreeMapAsValue(true, false);
}
/**
@@ -98,8 +98,26 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
*
* @throws Exception If failed.
*/
- public void testTreeMapBinaryComparator() throws Exception {
- checkTreeMap(true, true);
+ public void testTreeMapAsValueBinaryComparator() throws Exception {
+ checkTreeMapAsValue(true, true);
+ }
+
+ /**
+ * Test {@code TreeMap} data structure when used as key.
+ *
+ * @throws Exception If failed.
+ */
+ public void testTreeMapAsKeyNoComparator() throws Exception {
+ checkTreeMapAsKey(false);
+ }
+
+ /**
+ * Test {@code TreeMap} data structure with comparator when used as key.
+ *
+ * @throws Exception If failed.
+ */
+ public void testTreeMapAsKeyComparator() throws Exception {
+ checkTreeMapAsKey(true);
}
/**
@@ -110,22 +128,11 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
@SuppressWarnings("unchecked")
- private void checkTreeMap(boolean useBinary, boolean useComparator) throws Exception {
+ private void checkTreeMapAsValue(boolean useBinary, boolean useComparator) throws Exception {
// Populate map.
TreeMap<TestKey, Integer> map;
- if (useComparator) {
- map = new TreeMap<>(new TestKeyComparator());
-
- for (int i = 0; i < SIZE; i++)
- map.put(key(false, i), i);
- }
- else {
- map = new TreeMap<>();
-
- for (int i = 0; i < SIZE; i++)
- map.put(key(true, i), i);
- }
+ map = testMap(useComparator);
// Put and get value from cache.
cache().put(KEY, map);
@@ -147,6 +154,71 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
assertNull(resMap.comparator());
assertEquals(map, resMap);
+
+ cache().clear();
+ }
+
+ /**
+ * Check {@code TreeMap} data structure when used as key.
+ *
+ * @param useComparator Whether comparator should be used.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ private void checkTreeMapAsKey(boolean useComparator) throws Exception {
+ // Populate map.
+ TreeMap<TestKey, Integer> map;
+
+ map = testMap(useComparator);
+
+ // Put and get value from cache.
+ cache().put(map, KEY);
+
+ TreeMap<TestKey, Integer> resMap = (TreeMap<TestKey, Integer>)((Entry)cache().iterator().next()).getKey();
+
+ // Ensure content is correct.
+ if (useComparator)
+ assert resMap.comparator() instanceof TestKeyComparator;
+ else
+ assertNull(resMap.comparator());
+
+ assertEquals(map, resMap);
+
+ // Ensure value is correct.
+ Integer resSameMap = (Integer)cache().get(map);
+
+ assertEquals((Object)KEY, resSameMap);
+
+ Integer resIdenticalMap = (Integer)cache().get(testMap(useComparator));
+
+ assertEquals((Object)KEY, resIdenticalMap);
+
+ // Ensure wrong comparator is not accepted.
+ Integer resDifferentComp = (Integer)cache().get(testMap(!useComparator));
+
+ assertEquals(null, resDifferentComp);
+
+ cache().clear();
+ }
+
+ /** */
+ private TreeMap<TestKey, Integer> testMap(boolean useComp) {
+ TreeMap<TestKey, Integer> map;
+
+ if (useComp) {
+ map = new TreeMap<>(new TestKeyComparator());
+
+ for (int i = 0; i < SIZE; i++)
+ map.put(key(false, i), i);
+ }
+ else {
+ map = new TreeMap<>();
+
+ for (int i = 0; i < SIZE; i++)
+ map.put(key(true, i), i);
+ }
+
+ return map;
}
/**
@@ -154,8 +226,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
*
* @throws Exception If failed.
*/
- public void testTreeSetRegularNoComparator() throws Exception {
- checkTreeSet(false, false);
+ public void testTreeSetAsValueRegularNoComparator() throws Exception {
+ checkTreeSetAsValue(false, false);
}
/**
@@ -163,8 +235,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
*
* @throws Exception If failed.
*/
- public void testTreeSetRegularComparator() throws Exception {
- checkTreeSet(false, true);
+ public void testTreeSetAsValueRegularComparator() throws Exception {
+ checkTreeSetAsValue(false, true);
}
/**
@@ -172,8 +244,8 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
*
* @throws Exception If failed.
*/
- public void testTreeSetBinaryNoComparator() throws Exception {
- checkTreeSet(true, false);
+ public void testTreeSetAsValueBinaryNoComparator() throws Exception {
+ checkTreeSetAsValue(true, false);
}
/**
@@ -181,8 +253,26 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
*
* @throws Exception If failed.
*/
- public void testTreeSetBinaryComparator() throws Exception {
- checkTreeSet(true, true);
+ public void testTreeSetAsValueBinaryComparator() throws Exception {
+ checkTreeSetAsValue(true, true);
+ }
+
+ /**
+ * Test {@code TreeSet} data structure.
+ *
+ * @throws Exception If failed.
+ */
+ public void testTreeSetAsKeyNoComparator() throws Exception {
+ checkTreeSetAsKey(false);
+ }
+
+ /**
+ * Test {@code TreeSet} data structure with comparator.
+ *
+ * @throws Exception If failed.
+ */
+ public void testTreeSetAsKeyComparator() throws Exception {
+ checkTreeSetAsKey(true);
}
/**
@@ -193,22 +283,11 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
@SuppressWarnings("unchecked")
- private void checkTreeSet(boolean useBinary, boolean useComparator) throws Exception {
+ private void checkTreeSetAsValue(boolean useBinary, boolean useComparator) throws Exception {
// Populate set.
TreeSet<TestKey> set;
- if (useComparator) {
- set = new TreeSet<>(new TestKeyComparator());
-
- for (int i = 0; i < SIZE; i++)
- set.add(key(false, i));
- }
- else {
- set = new TreeSet<>();
-
- for (int i = 0; i < SIZE; i++)
- set.add(key(true, i));
- }
+ set = testSet(useComparator);
// Put and get value from cache.
cache().put(KEY, set);
@@ -230,6 +309,73 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
assertNull(resSet.comparator());
assertEquals(set, resSet);
+
+ cache().clear();
+ }
+
+ /**
+ * Check {@code TreeSet} data structure when used as key.
+ *
+ * @param useComp Whether comparator should be used.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ private void checkTreeSetAsKey(boolean useComp) throws Exception {
+ // Populate set.
+ TreeSet<TestKey> set;
+
+ set = testSet(useComp);
+
+ // Put and get value from cache.
+ cache().put(set, KEY);
+
+ TreeSet<TestKey> resSet = (TreeSet<TestKey>)((Entry)cache().iterator().next()).getKey();
+
+ // Ensure content is correct.
+ if (useComp)
+ assert resSet.comparator() instanceof TestKeyComparator;
+ else
+ assertNull(resSet.comparator());
+
+ assertEquals(set, resSet);
+
+ // Ensure value is correct.
+ Integer resSameMap = (Integer)cache().get(set);
+
+ assertEquals((Object)KEY, resSameMap);
+
+ Integer resIdenticalMap = (Integer)cache().get(testSet(useComp));
+
+ assertEquals((Object)KEY, resIdenticalMap);
+
+ // Ensure wrong comparator is not accepted.
+ Integer resDifferentComp = (Integer)cache().get(testSet(!useComp));
+
+ assertEquals(null, resDifferentComp);
+
+ assertEquals(set, resSet);
+
+ cache().clear();
+ }
+
+ /** */
+ private TreeSet<TestKey> testSet(boolean useComp) {
+ TreeSet<TestKey> set;
+
+ if (useComp) {
+ set = new TreeSet<>(new TestKeyComparator());
+
+ for (int i = 0; i < SIZE; i++)
+ set.add(key(false, i));
+ }
+ else {
+ set = new TreeSet<>();
+
+ for (int i = 0; i < SIZE; i++)
+ set.add(key(true, i));
+ }
+
+ return set;
}
/**
@@ -308,6 +454,23 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
public int id() {
return id;
}
+
+ /** */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ TestKey key = (TestKey)o;
+
+ return id == key.id;
+ }
+
+ /** */
+ @Override public int hashCode() {
+ return id;
+ }
}
/**
@@ -337,5 +500,21 @@ public class BinaryTreeSelfTest extends GridCommonAbstractTest {
@Override public int compare(TestKey o1, TestKey o2) {
return o1.id() - o2.id();
}
+
+ /** */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ return true;
+ }
+
+ /** */
+ @Override public int hashCode() {
+ return 13;
+ }
}
}
[17/50] [abbrv] ignite git commit: IGNITE-7860: JDBC thin: changed
default socket buffer size to 64Kb. This closes #3600.
Posted by sb...@apache.org.
IGNITE-7860: JDBC thin: changed default socket buffer size to 64Kb. This closes #3600.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1142a368
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1142a368
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1142a368
Branch: refs/heads/ignite-zk
Commit: 1142a368187a07fe91993edebb40bd027c7cb72a
Parents: 3cb9e25
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Tue Mar 13 12:37:29 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Tue Mar 13 12:37:29 2018 +0300
----------------------------------------------------------------------
.../ignite/jdbc/thin/JdbcThinConnectionSelfTest.java | 10 ++++++----
.../internal/jdbc/thin/ConnectionPropertiesImpl.java | 7 +++++--
2 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1142a368/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
index ad98683..bdaa107 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
@@ -155,6 +155,8 @@ public class JdbcThinConnectionSelfTest extends JdbcThinAbstractSelfTest {
* @throws Exception If failed.
*/
public void testSocketBuffers() throws Exception {
+ final int dfltDufSize = 64 * 1024;
+
assertInvalid("jdbc:ignite:thin://127.0.0.1?socketSendBuffer=-1",
"Property cannot be lower than 0 [name=socketSendBuffer, value=-1]");
@@ -162,18 +164,18 @@ public class JdbcThinConnectionSelfTest extends JdbcThinAbstractSelfTest {
"Property cannot be lower than 0 [name=socketReceiveBuffer, value=-1]");
try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1")) {
- assertEquals(0, io(conn).connectionProperties().getSocketSendBuffer());
- assertEquals(0, io(conn).connectionProperties().getSocketReceiveBuffer());
+ assertEquals(dfltDufSize, io(conn).connectionProperties().getSocketSendBuffer());
+ assertEquals(dfltDufSize, io(conn).connectionProperties().getSocketReceiveBuffer());
}
// Note that SO_* options are hints, so we check that value is equals to either what we set or to default.
try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?socketSendBuffer=1024")) {
assertEquals(1024, io(conn).connectionProperties().getSocketSendBuffer());
- assertEquals(0, io(conn).connectionProperties().getSocketReceiveBuffer());
+ assertEquals(dfltDufSize, io(conn).connectionProperties().getSocketReceiveBuffer());
}
try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?socketReceiveBuffer=1024")) {
- assertEquals(0, io(conn).connectionProperties().getSocketSendBuffer());
+ assertEquals(dfltDufSize, io(conn).connectionProperties().getSocketSendBuffer());
assertEquals(1024, io(conn).connectionProperties().getSocketReceiveBuffer());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1142a368/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
index 471381b..060c5bf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
@@ -38,6 +38,9 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
/** Prefix for property names. */
public static final String PROP_PREFIX = "ignite.jdbc.";
+ /** Default socket buffer size. */
+ private static final int DFLT_SOCK_BUFFER_SIZE = 64 * 1024;
+
/** Host name property. */
private StringProperty host = new StringProperty(
"host", "Ignite node IP to connect", null, null, true,
@@ -80,12 +83,12 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
/** Socket send buffer size property. */
private IntegerProperty socketSendBuffer = new IntegerProperty(
"socketSendBuffer", "Socket send buffer size",
- 0, false, 0, Integer.MAX_VALUE);
+ DFLT_SOCK_BUFFER_SIZE, false, 0, Integer.MAX_VALUE);
/** Socket receive buffer size property. */
private IntegerProperty socketReceiveBuffer = new IntegerProperty(
"socketReceiveBuffer", "Socket send buffer size",
- 0, false, 0, Integer.MAX_VALUE);
+ DFLT_SOCK_BUFFER_SIZE, false, 0, Integer.MAX_VALUE);
/** Executes update queries on ignite server nodes flag. */
private BooleanProperty skipReducerOnUpdate = new BooleanProperty(
[46/50] [abbrv] ignite git commit: IGNITE-7964 rmvId is stored to
MetaStorage metapage during operations - Fixes #3645.
Posted by sb...@apache.org.
IGNITE-7964 rmvId is stored to MetaStorage metapage during operations - Fixes #3645.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/892d8c73
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/892d8c73
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/892d8c73
Branch: refs/heads/ignite-zk
Commit: 892d8c739205c2b87598568042f0cffa3004d57b
Parents: f4ac81a
Author: Sergey Chugunov <se...@gmail.com>
Authored: Fri Mar 16 19:24:17 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Mar 16 19:24:17 2018 +0300
----------------------------------------------------------------------
.../GridCacheDatabaseSharedManager.java | 4 +-
.../persistence/metastorage/MetaStorage.java | 57 +++++--
.../metastorage/IgniteMetaStorageBasicTest.java | 147 +++++++++++++++++++
.../ignite/testsuites/IgnitePdsTestSuite.java | 4 +
4 files changed, 199 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/892d8c73/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 997f89a..45d9de5 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -553,7 +553,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
try {
restoreMemory(status, true, storePageMem);
- metaStorage = new MetaStorage(cctx.wal(), regCfg, memMetrics, true);
+ metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true);
metaStorage.init(this);
@@ -724,7 +724,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
cctx.pageStore().initializeForMetastorage();
- metaStorage = new MetaStorage(cctx.wal(), dataRegionMap.get(METASTORE_DATA_REGION_NAME),
+ metaStorage = new MetaStorage(cctx, dataRegionMap.get(METASTORE_DATA_REGION_NAME),
(DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME));
WALPointer restore = restoreMemory(status);
http://git-wip-us.apache.org/repos/asf/ignite/blob/892d8c73/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index d57b939..d225659 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
@@ -31,6 +32,7 @@ import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.IncompleteObject;
import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
@@ -79,6 +81,9 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
private final DataRegion dataRegion;
/** */
+ private final IgniteLogger log;
+
+ /** */
private MetastorageTree tree;
/** */
@@ -109,17 +114,18 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
private final Marshaller marshaller = new JdkMarshaller();
/** */
- public MetaStorage(IgniteWriteAheadLogManager wal, DataRegion dataRegion, DataRegionMetricsImpl regionMetrics,
+ public MetaStorage(GridCacheSharedContext cctx, DataRegion dataRegion, DataRegionMetricsImpl regionMetrics,
boolean readOnly) {
- this.wal = wal;
+ wal = cctx.wal();
this.dataRegion = dataRegion;
this.regionMetrics = regionMetrics;
this.readOnly = readOnly;
+ log = cctx.logger(getClass());
}
/** */
- public MetaStorage(IgniteWriteAheadLogManager wal, DataRegion memPlc, DataRegionMetricsImpl memMetrics) {
- this(wal, memPlc, memMetrics, false);
+ public MetaStorage(GridCacheSharedContext cctx, DataRegion memPlc, DataRegionMetricsImpl memMetrics) {
+ this(cctx, memPlc, memMetrics, false);
}
/** */
@@ -309,6 +315,8 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
treeRoot = new RootPage(new FullPageId(io.getTreeRoot(pageAddr), METASTORAGE_CACHE_ID), false);
reuseListRoot = new RootPage(new FullPageId(io.getReuseListRoot(pageAddr), METASTORAGE_CACHE_ID), false);
+
+ rmvId.set(io.getGlobalRemoveId(pageAddr));
}
finally {
pageMem.readUnlock(METASTORAGE_CACHE_ID, partId, partMetaPage);
@@ -354,6 +362,8 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
treeRoot = io.getTreeRoot(pageAddr);
reuseListRoot = io.getReuseListRoot(pageAddr);
+ rmvId.set(io.getGlobalRemoveId(pageAddr));
+
assert PageIdUtils.flag(treeRoot) == PageMemory.FLAG_DATA :
U.hexLong(treeRoot) + ", part=" + partId + ", METASTORAGE_CACHE_ID=" + METASTORAGE_CACHE_ID;
assert PageIdUtils.flag(reuseListRoot) == PageMemory.FLAG_DATA :
@@ -384,21 +394,46 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
@Override public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
freeList.saveMetadata();
- MetastorageRowStore rowStore = tree.rowStore();
-
- saveStoreMetadata(rowStore, ctx);
+ saveStoreMetadata();
}
/**
- * @param rowStore Store to save metadata.
* @throws IgniteCheckedException If failed.
*/
- private void saveStoreMetadata(MetastorageRowStore rowStore, Context ctx) throws IgniteCheckedException {
- FreeListImpl freeList = (FreeListImpl)rowStore.freeList();
+ private void saveStoreMetadata() throws IgniteCheckedException {
+ PageMemoryEx pageMem = (PageMemoryEx) pageMemory();
- freeList.saveMetadata();
+ int partId = 0;
+
+ long partMetaId = pageMem.partitionMetaPageId(METASTORAGE_CACHE_ID, partId);
+ long partMetaPage = pageMem.acquirePage(METASTORAGE_CACHE_ID, partMetaId);
+
+ try {
+ long partMetaPageAddr = pageMem.writeLock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+
+ if (partMetaPageAddr == 0L) {
+ U.warn(log, "Failed to acquire write lock for meta page [metaPage=" + partMetaPage + ']');
+
+ return;
+ }
+
+ boolean changed = false;
+
+ try {
+ PagePartitionMetaIO io = PageIO.getPageIO(partMetaPageAddr);
+
+ changed |= io.setGlobalRemoveId(partMetaPageAddr, rmvId.get());
+ }
+ finally {
+ pageMem.writeUnlock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage, null, changed);
+ }
+ }
+ finally {
+ pageMem.releasePage(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+ }
}
+ /** */
public void applyUpdate(String key, byte[] value) throws IgniteCheckedException {
if (readOnly) {
if (lastUpdates == null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/892d8c73/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
new file mode 100644
index 0000000..1837515
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import java.io.Serializable;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ * Single place to add for basic MetaStorage tests.
+ */
+public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ DataStorageConfiguration storageCfg = new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setMaxSize(100 * 1024 * 1024)
+ .setPersistenceEnabled(true)
+ )
+ .setWalMode(WALMode.LOG_ONLY);
+
+ cfg.setDataStorageConfiguration(storageCfg);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(discoSpi);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ cleanPersistenceDir();
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * Verifies that MetaStorage after massive amounts of keys stored and updated keys restores its state successfully
+ * after restart.
+ *
+ * See <a hfer="https://issues.apache.org/jira/browse/IGNITE-7964" target="_top">IGNITE-7964</a> for more context
+ * about this test.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMetaStorageMassivePutUpdateRestart() throws Exception {
+ IgniteEx ig = startGrid(0);
+
+ ig.cluster().active(true);
+
+ final byte KEYS_CNT = 100;
+ final String KEY_PREFIX = "test.key.";
+ final String NEW_VAL_PREFIX = "new.val.";
+ final String UPDATED_VAL_PREFIX = "updated.val.";
+
+ loadKeys(ig, KEYS_CNT, KEY_PREFIX, NEW_VAL_PREFIX, UPDATED_VAL_PREFIX);
+
+ stopGrid(0);
+
+ ig = startGrid(0);
+
+ ig.cluster().active(true);
+
+ verifyKeys(ig, KEYS_CNT, KEY_PREFIX, UPDATED_VAL_PREFIX);
+ }
+
+ /** */
+ private void loadKeys(IgniteEx ig,
+ byte keysCnt,
+ String keyPrefix,
+ String newValPrefix,
+ String updatedValPrefix
+ ) throws IgniteCheckedException {
+ IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database();
+
+ MetaStorage metaStorage = db.metaStorage();
+
+ db.checkpointReadLock();
+ try {
+ for (byte i = 0; i < keysCnt; i++)
+ metaStorage.write(keyPrefix + i, newValPrefix + i);
+
+ for (byte i = 0; i < keysCnt; i++)
+ metaStorage.write(keyPrefix + i, updatedValPrefix + i);
+ }
+ finally {
+ db.checkpointReadUnlock();
+ }
+ }
+
+ /** */
+ private void verifyKeys(IgniteEx ig,
+ byte keysCnt,
+ String keyPrefix,
+ String valPrefix
+ ) throws IgniteCheckedException {
+ MetaStorage metaStorage = ig.context().cache().context().database().metaStorage();
+
+ for (byte i = 0; i < keysCnt; i++) {
+ Serializable val = metaStorage.read(keyPrefix + i);
+
+ Assert.assertEquals(valPrefix + i, val);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/892d8c73/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index a36f35b..a9668e7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -26,6 +26,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsDataR
import org.apache.ignite.internal.processors.cache.persistence.db.file.DefaultPageSizeBackwardsCompatibilityTest;
import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCheckpointSimulationWithRealCpDisabledTest;
import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsPageReplacementTest;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.IgniteMetaStorageBasicTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreePageMemoryImplTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreeReuseListPageMemoryImplTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.FillFactorMetricTest;
@@ -114,5 +115,8 @@ public class IgnitePdsTestSuite extends TestSuite {
suite.addTestSuite(IgnitePdsDataRegionMetricsTest.class);
suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
+
+ //MetaStorage
+ suite.addTestSuite(IgniteMetaStorageBasicTest.class);
}
}
[25/50] [abbrv] ignite git commit: IGNITE-7253: JDBC thin driver:
implemented streaming. This closes #3499. This closes #3591.
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index f666cdd..bde9427 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -263,7 +263,8 @@ public class GridQueryProcessor extends GridProcessorAdapter {
idx.cancelAllQueries();
return;
- } catch (InterruptedException ignored) {
+ }
+ catch (InterruptedException ignored) {
U.warn(log, "Interrupted while waiting for active queries cancellation.");
Thread.currentThread().interrupt();
@@ -1974,13 +1975,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
*/
public List<FieldsQueryCursor<List<?>>> querySqlFields(final SqlFieldsQuery qry, final boolean keepBinary,
final boolean failOnMultipleStmts) {
- return querySqlFields(null, qry, keepBinary, failOnMultipleStmts);
- }
-
- @SuppressWarnings("unchecked")
- public FieldsQueryCursor<List<?>> querySqlFields(final GridCacheContext<?,?> cctx, final SqlFieldsQuery qry,
- final boolean keepBinary) {
- return querySqlFields(cctx, qry, keepBinary, true).get(0);
+ return querySqlFields(null, qry, null, keepBinary, failOnMultipleStmts);
}
/**
@@ -1991,7 +1986,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
* @return Cursor.
*/
public FieldsQueryCursor<List<?>> querySqlFields(final SqlFieldsQuery qry, final boolean keepBinary) {
- return querySqlFields(null, qry, keepBinary, true).get(0);
+ return querySqlFields(null, qry, null, keepBinary, true).get(0);
}
/**
@@ -1999,14 +1994,16 @@ public class GridQueryProcessor extends GridProcessorAdapter {
*
* @param cctx Cache context.
* @param qry Query.
+ * @param cliCtx Client context.
* @param keepBinary Keep binary flag.
* @param failOnMultipleStmts If {@code true} the method must throws exception when query contains
* more then one SQL statement.
* @return Cursor.
*/
@SuppressWarnings("unchecked")
- public List<FieldsQueryCursor<List<?>>> querySqlFields(@Nullable final GridCacheContext<?,?> cctx,
- final SqlFieldsQuery qry, final boolean keepBinary, final boolean failOnMultipleStmts) {
+ public List<FieldsQueryCursor<List<?>>> querySqlFields(@Nullable final GridCacheContext<?, ?> cctx,
+ final SqlFieldsQuery qry, final SqlClientContext cliCtx, final boolean keepBinary,
+ final boolean failOnMultipleStmts) {
checkxEnabled();
validateSqlFieldsQuery(qry);
@@ -2034,7 +2031,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
GridQueryCancel cancel = new GridQueryCancel();
List<FieldsQueryCursor<List<?>>> res =
- idx.querySqlFields(schemaName, qry, keepBinary, failOnMultipleStmts, cancel);
+ idx.querySqlFields(schemaName, qry, cliCtx, keepBinary, failOnMultipleStmts, cancel);
if (cctx != null)
sendQueryExecutedEvent(qry.getSql(), qry.getArgs(), cctx);
@@ -2073,7 +2070,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
* @param schemaName Schema name.
* @param streamer Data streamer.
* @param qry Query.
- * @return Iterator.
+ * @return Update counter.
*/
public long streamUpdateQuery(@Nullable final String cacheName, final String schemaName,
final IgniteDataStreamer<?, ?> streamer, final String qry, final Object[] args) {
@@ -2100,6 +2097,33 @@ public class GridQueryProcessor extends GridProcessorAdapter {
}
/**
+ * @param schemaName Schema name.
+ * @param cliCtx Client context.
+ * @param qry Query.
+ * @param args Query arguments.
+ * @return Update counters.
+ */
+ public List<Long> streamBatchedUpdateQuery(final String schemaName, final SqlClientContext cliCtx,
+ final String qry, final List<Object[]> args) {
+ if (!busyLock.enterBusy())
+ throw new IllegalStateException("Failed to execute query (grid is stopping).");
+
+ try {
+ return executeQuery(GridCacheQueryType.SQL_FIELDS, qry, null, new IgniteOutClosureX<List<Long>>() {
+ @Override public List<Long> applyx() throws IgniteCheckedException {
+ return idx.streamBatchedUpdateQuery(schemaName, qry, args, cliCtx);
+ }
+ }, true);
+ }
+ catch (IgniteCheckedException e) {
+ throw new CacheException(e);
+ }
+ finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
* Execute distributed SQL query.
*
* @param cctx Cache context.
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/query/SqlClientContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/SqlClientContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/SqlClientContext.java
new file mode 100644
index 0000000..e8c2932
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/SqlClientContext.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Container for connection properties passed by various drivers (JDBC drivers, possibly ODBC) having notion of an
+ * <b>SQL connection</b> - Ignite basically does not have one.<p>
+ * Also contains anything that a driver may need to share between threads processing queries of logically same client -
+ * see JDBC thin driver
+ */
+public class SqlClientContext implements AutoCloseable {
+ /** Kernal context. */
+ private final GridKernalContext ctx;
+
+ /** Distributed joins flag. */
+ private final boolean distributedJoins;
+
+ /** Enforce join order flag. */
+ private final boolean enforceJoinOrder;
+
+ /** Collocated flag. */
+ private final boolean collocated;
+
+ /** Replicated caches only flag. */
+ private final boolean replicatedOnly;
+
+ /** Lazy query execution flag. */
+ private final boolean lazy;
+
+ /** Skip reducer on update flag. */
+ private final boolean skipReducerOnUpdate;
+
+ /** Allow overwrites for duplicate keys on streamed {@code INSERT}s. */
+ private boolean streamAllowOverwrite;
+
+ /** Parallel ops count per node for data streamer. */
+ private int streamNodeParOps;
+
+ /** Node buffer size for data streamer. */
+ private int streamNodeBufSize;
+
+ /** Auto flush frequency for streaming. */
+ private long streamFlushTimeout;
+
+ /** Streamers for various caches. */
+ private Map<String, IgniteDataStreamer<?, ?>> streamers;
+
+ /** Logger. */
+ private final IgniteLogger log;
+
+ /**
+ * @param ctx Kernal context.
+ * @param distributedJoins Distributed joins flag.
+ * @param enforceJoinOrder Enforce join order flag.
+ * @param collocated Collocated flag.
+ * @param replicatedOnly Replicated caches only flag.
+ * @param lazy Lazy query execution flag.
+ * @param skipReducerOnUpdate Skip reducer on update flag.
+ */
+ public SqlClientContext(GridKernalContext ctx, boolean distributedJoins, boolean enforceJoinOrder,
+ boolean collocated, boolean replicatedOnly, boolean lazy, boolean skipReducerOnUpdate) {
+ this.ctx = ctx;
+ this.distributedJoins = distributedJoins;
+ this.enforceJoinOrder = enforceJoinOrder;
+ this.collocated = collocated;
+ this.replicatedOnly = replicatedOnly;
+ this.lazy = lazy;
+ this.skipReducerOnUpdate = skipReducerOnUpdate;
+
+ log = ctx.log(SqlClientContext.class.getName());
+ }
+
+ /**
+ * Turn on streaming on this client context.
+ *
+ * @param allowOverwrite Whether streaming should overwrite existing values.
+ * @param flushFreq Flush frequency for streamers.
+ * @param perNodeBufSize Per node streaming buffer size.
+ * @param perNodeParOps Per node streaming parallel operations number.
+ */
+ public void enableStreaming(boolean allowOverwrite, long flushFreq, int perNodeBufSize, int perNodeParOps) {
+ if (isStream())
+ return;
+
+ streamers = new HashMap<>();
+
+ this.streamAllowOverwrite = allowOverwrite;
+ this.streamFlushTimeout = flushFreq;
+ this.streamNodeBufSize = perNodeBufSize;
+ this.streamNodeParOps = perNodeParOps;
+ }
+
+ /**
+ * Turn off streaming on this client context - with closing all open streamers, if any.
+ */
+ public void disableStreaming() {
+ if (!isStream())
+ return;
+
+ Iterator<IgniteDataStreamer<?, ?>> it = streamers.values().iterator();
+
+ while (it.hasNext()) {
+ IgniteDataStreamer<?, ?> streamer = it.next();
+
+ U.close(streamer, log);
+
+ it.remove();
+ }
+
+ streamers = null;
+ }
+
+ /**
+ * @return Collocated flag.
+ */
+ public boolean isCollocated() {
+ return collocated;
+ }
+
+ /**
+ * @return Distributed joins flag.
+ */
+ public boolean isDistributedJoins() {
+ return distributedJoins;
+ }
+
+ /**
+ * @return Enforce join order flag.
+ */
+ public boolean isEnforceJoinOrder() {
+ return enforceJoinOrder;
+ }
+
+ /**
+ * @return Replicated caches only flag.
+ */
+ public boolean isReplicatedOnly() {
+ return replicatedOnly;
+ }
+
+ /**
+ * @return Lazy query execution flag.
+ */
+ public boolean isLazy() {
+ return lazy;
+ }
+
+ /**
+ * @return Skip reducer on update flag,
+ */
+ public boolean isSkipReducerOnUpdate() {
+ return skipReducerOnUpdate;
+ }
+
+ /**
+ * @return Streaming state flag (on or off).
+ */
+ public boolean isStream() {
+ return streamers != null;
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @return Streamer for given cache.
+ */
+ public IgniteDataStreamer<?, ?> streamerForCache(String cacheName) {
+ if (streamers == null)
+ return null;
+
+ IgniteDataStreamer<?, ?> res = streamers.get(cacheName);
+
+ if (res != null)
+ return res;
+
+ res = ctx.grid().dataStreamer(cacheName);
+
+ res.autoFlushFrequency(streamFlushTimeout);
+
+ res.allowOverwrite(streamAllowOverwrite);
+
+ if (streamNodeBufSize > 0)
+ res.perNodeBufferSize(streamNodeBufSize);
+
+ if (streamNodeParOps > 0)
+ res.perNodeParallelOperations(streamNodeParOps);
+
+ streamers.put(cacheName, res);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws Exception {
+ if (streamers == null)
+ return;
+
+ for (IgniteDataStreamer<?, ?> s : streamers.values())
+ U.close(s, log);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java
index 9a2a865..26b2263 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java
@@ -17,11 +17,10 @@
package org.apache.ignite.internal.sql;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.util.typedef.F;
-
import java.lang.reflect.Field;
import java.util.HashSet;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.util.typedef.F;
/**
* SQL keyword constants.
@@ -30,9 +29,15 @@ public class SqlKeyword {
/** Keyword: ALTER. */
public static final String ALTER = "ALTER";
+ /** Keyword: ALLOW_OVERWRITE */
+ public static final String ALLOW_OVERWRITE = "ALLOW_OVERWRITE";
+
/** Keyword: ASC. */
public static final String ASC = "ASC";
+ /** Keyword: BATCH_SIZE */
+ public static final String BATCH_SIZE = "BATCH_SIZE";
+
/** Keyword: BIGINT */
public static final String BIGINT = "BIGINT";
@@ -96,6 +101,9 @@ public class SqlKeyword {
/** Keyword: FLOAT8. */
public static final String FLOAT8 = "FLOAT8";
+ /** Keyword: FLUSH_FREQUENCY. */
+ public static final String FLUSH_FREQUENCY = "FLUSH_FREQUENCY";
+
/** Keyword: FORMAT. */
public static final String FORMAT = "FORMAT";
@@ -168,9 +176,18 @@ public class SqlKeyword {
/** Keyword: NVARCHAR2. */
public static final String NVARCHAR2 = "NVARCHAR2";
+ /** Keyword: OFF. */
+ public static final String OFF = "OFF";
+
/** Keyword: ON. */
public static final String ON = "ON";
+ /** Keyword: PER_NODE_PARALLEL_OPERATIONS. */
+ public static final String PER_NODE_PARALLEL_OPERATIONS = "PER_NODE_PARALLEL_OPERATIONS";
+
+ /** Keyword: PER_NODE_BUFFER_SIZE. */
+ public static final String PER_NODE_BUFFER_SIZE = "PER_NODE_BUFFER_SIZE";
+
/** Keyword: PRECISION. */
public static final String PRECISION = "PRECISION";
@@ -183,6 +200,9 @@ public class SqlKeyword {
/** Keyword: RESTRICT. */
public static final String RESTRICT = "RESTRICT";
+ /** Keyword: SET. */
+ public static final String SET = "SET";
+
/** Keyword: SIGNED. */
public static final String SIGNED = "SIGNED";
@@ -195,6 +215,9 @@ public class SqlKeyword {
/** Keyword: SPATIAL. */
public static final String SPATIAL = "SPATIAL";
+ /** Keyword: STREAMING. */
+ public static final String STREAMING = "STREAMING";
+
/** Keyword: TABLE. */
public static final String TABLE = "TABLE";
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java
index 0627def..da6d28e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java
@@ -22,6 +22,7 @@ import org.apache.ignite.internal.sql.command.SqlBulkLoadCommand;
import org.apache.ignite.internal.sql.command.SqlCommand;
import org.apache.ignite.internal.sql.command.SqlCreateIndexCommand;
import org.apache.ignite.internal.sql.command.SqlDropIndexCommand;
+import org.apache.ignite.internal.sql.command.SqlSetStreamingCommand;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.sql.SqlKeyword.ALTER;
@@ -31,7 +32,9 @@ import static org.apache.ignite.internal.sql.SqlKeyword.DROP;
import static org.apache.ignite.internal.sql.SqlKeyword.HASH;
import static org.apache.ignite.internal.sql.SqlKeyword.INDEX;
import static org.apache.ignite.internal.sql.SqlKeyword.PRIMARY;
+import static org.apache.ignite.internal.sql.SqlKeyword.SET;
import static org.apache.ignite.internal.sql.SqlKeyword.SPATIAL;
+import static org.apache.ignite.internal.sql.SqlKeyword.STREAMING;
import static org.apache.ignite.internal.sql.SqlKeyword.TABLE;
import static org.apache.ignite.internal.sql.SqlKeyword.UNIQUE;
import static org.apache.ignite.internal.sql.SqlParserUtils.errorUnexpectedToken;
@@ -110,6 +113,11 @@ public class SqlParser {
break;
+ case SET:
+ cmd = processSet();
+
+ break;
+
case ALTER:
cmd = processAlter();
}
@@ -122,7 +130,7 @@ public class SqlParser {
return cmd;
}
else
- throw errorUnexpectedToken(lex, CREATE, DROP, COPY, ALTER);
+ throw errorUnexpectedToken(lex, CREATE, DROP, ALTER, COPY, SET);
case QUOTED:
case MINUS:
@@ -137,6 +145,22 @@ public class SqlParser {
}
/**
+ * Process SET keyword.
+ *
+ * @return Command.
+ */
+ private SqlCommand processSet() {
+ if (lex.shift() && lex.tokenType() == SqlLexerTokenType.DEFAULT) {
+ switch (lex.token()) {
+ case STREAMING:
+ return new SqlSetStreamingCommand().parse(lex);
+ }
+ }
+
+ throw errorUnexpectedToken(lex, STREAMING);
+ }
+
+ /**
* Processes COPY command.
*
* @return The {@link SqlBulkLoadCommand} command.
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParserUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParserUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParserUtils.java
index 829c48c..9ed75ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParserUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParserUtils.java
@@ -123,6 +123,28 @@ public class SqlParserUtils {
}
/**
+ * Parse boolean parameter value based on presence of tokens 1, 0, ON, OFF. Not that this is not
+ * and is not intended to be routine for parsing a boolean literal from TRUE/FALSE.
+ * @param lex Lexer.
+ * @return Boolean parameter value.
+ */
+ public static boolean parseBoolean(SqlLexer lex) {
+ if (lex.shift() && lex.tokenType() == SqlLexerTokenType.DEFAULT) {
+ switch (lex.token()) {
+ case SqlKeyword.ON:
+ case "1":
+ return true;
+
+ case SqlKeyword.OFF:
+ case "0":
+ return false;
+ }
+ }
+
+ throw errorUnexpectedToken(lex, SqlKeyword.ON, SqlKeyword.OFF, "1", "0");
+ }
+
+ /**
* Process name.
*
* @param lex Lexer.
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/sql/command/SqlSetStreamingCommand.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/sql/command/SqlSetStreamingCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/sql/command/SqlSetStreamingCommand.java
new file mode 100644
index 0000000..c492c61
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/sql/command/SqlSetStreamingCommand.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.sql.command;
+
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.sql.SqlKeyword;
+import org.apache.ignite.internal.sql.SqlLexer;
+import org.apache.ignite.internal.sql.SqlLexerTokenType;
+import org.apache.ignite.internal.sql.SqlParseException;
+
+import static org.apache.ignite.internal.sql.SqlParserUtils.error;
+import static org.apache.ignite.internal.sql.SqlParserUtils.errorUnexpectedToken;
+import static org.apache.ignite.internal.sql.SqlParserUtils.parseBoolean;
+import static org.apache.ignite.internal.sql.SqlParserUtils.parseInt;
+
+/**
+ * SET STREAMING command.
+ */
+public class SqlSetStreamingCommand implements SqlCommand {
+ /** Default batch size for driver. */
+ private final static int DFLT_STREAM_BATCH_SIZE = IgniteDataStreamer.DFLT_PER_NODE_BUFFER_SIZE * 4;
+
+ /** Whether streaming must be turned on or off by this command. */
+ private boolean turnOn;
+
+ /** Whether existing values should be overwritten on keys duplication. */
+ private boolean allowOverwrite;
+
+ /** Batch size for driver. */
+ private int batchSize = DFLT_STREAM_BATCH_SIZE;
+
+ /** Per node number of parallel operations. */
+ private int perNodeParOps;
+
+ /** Per node buffer size. */
+ private int perNodeBufSize;
+
+ /** Streamer flush timeout. */
+ private long flushFreq;
+
+ /** {@inheritDoc} */
+ @Override public SqlCommand parse(SqlLexer lex) {
+ turnOn = parseBoolean(lex);
+
+ while (lex.lookAhead().tokenType() == SqlLexerTokenType.DEFAULT) {
+ switch (lex.lookAhead().token()) {
+ case SqlKeyword.BATCH_SIZE:
+ lex.shift();
+
+ checkOffLast(lex);
+
+ batchSize = parseInt(lex);
+
+ if (batchSize <= 0)
+ throw error(lex, "Invalid batch size (must be positive).");
+
+ break;
+
+ case SqlKeyword.PER_NODE_BUFFER_SIZE:
+ lex.shift();
+
+ checkOffLast(lex);
+
+ perNodeBufSize = parseInt(lex);
+
+ if (perNodeBufSize <= 0)
+ throw error(lex, "Invalid per node buffer size (must be positive).");
+
+ break;
+
+ case SqlKeyword.PER_NODE_PARALLEL_OPERATIONS:
+ lex.shift();
+
+ checkOffLast(lex);
+
+ perNodeParOps = parseInt(lex);
+
+ if (perNodeParOps <= 0)
+ throw error(lex, "Invalid per node parallel operations number (must be positive).");
+
+ break;
+
+ case SqlKeyword.ALLOW_OVERWRITE:
+ lex.shift();
+
+ checkOffLast(lex);
+
+ allowOverwrite = parseBoolean(lex);
+
+ break;
+
+ case SqlKeyword.FLUSH_FREQUENCY:
+ lex.shift();
+
+ checkOffLast(lex);
+
+ flushFreq = parseInt(lex);
+
+ if (flushFreq <= 0)
+ throw error(lex, "Invalid flush frequency (must be positive).");
+
+ break;
+
+ default:
+ return this;
+ }
+ }
+
+ return this;
+ }
+
+ /**
+ * Throw an unexpected token exception if this command turns streaming off.
+ * @param lex Lexer to take unexpected token from.
+ * @throws SqlParseException if {@link #turnOn} is {@code false}.
+ */
+ private void checkOffLast(SqlLexer lex) throws SqlParseException {
+ if (!turnOn) {
+ assert lex.tokenType() == SqlLexerTokenType.DEFAULT;
+
+ throw errorUnexpectedToken(lex);
+ }
+ }
+
+ /**
+ * @return Whether streaming must be turned on or off by this command.
+ */
+ public boolean isTurnOn() {
+ return turnOn;
+ }
+
+ /**
+ * @return Whether existing values should be overwritten on keys duplication.
+ */
+ public boolean allowOverwrite() {
+ return allowOverwrite;
+ }
+
+ /**
+ * @return Batch size for driver.
+ */
+ public int batchSize() {
+ return batchSize;
+ }
+
+ /**
+ * @return Per node number of parallel operations.
+ */
+ public int perNodeParallelOperations() {
+ return perNodeParOps;
+ }
+
+ /**
+ * @return Per node streamer buffer size.
+ */
+ public int perNodeBufferSize() {
+ return perNodeBufSize;
+ }
+
+ /**
+ * @return Streamer flush timeout
+ */
+ public long flushFrequency() {
+ return flushFreq;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String schemaName() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void schemaName(String schemaName) {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
index df27c5f..6d7e9ae 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
@@ -52,6 +52,7 @@ import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
import org.apache.ignite.internal.processors.query.QueryField;
import org.apache.ignite.internal.processors.query.QueryIndexDescriptorImpl;
+import org.apache.ignite.internal.processors.query.SqlClientContext;
import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitor;
import org.apache.ignite.internal.util.GridSpinBusyLock;
import org.apache.ignite.internal.util.lang.GridCloseableIterator;
@@ -244,12 +245,18 @@ public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractT
/** {@inheritDoc} */
@Override public List<FieldsQueryCursor<List<?>>> querySqlFields(String schemaName, SqlFieldsQuery qry,
- boolean keepBinary, boolean failOnMultipleStmts, GridQueryCancel cancel) {
+ SqlClientContext cliCtx, boolean keepBinary, boolean failOnMultipleStmts, GridQueryCancel cancel) {
return null;
}
/** {@inheritDoc} */
- @Override public long streamUpdateQuery(String spaceName, String qry, @Nullable Object[] params,
+ @Override public List<Long> streamBatchedUpdateQuery(String schemaName, String qry, List<Object[]> params,
+ SqlClientContext cliCtx) throws IgniteCheckedException {
+ return Collections.emptyList();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long streamUpdateQuery(String schemaName, String qry, @Nullable Object[] params,
IgniteDataStreamer<?, ?> streamer) throws IgniteCheckedException {
return 0;
}
@@ -372,8 +379,8 @@ public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractT
}
/** {@inheritDoc} */
- @Override public boolean isInsertStatement(PreparedStatement nativeStmt) {
- return false;
+ @Override public void checkStatementStreamable(PreparedStatement nativeStmt) {
+ // No-op.
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/test/java/org/apache/ignite/internal/sql/SqlParserSetStreamingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/sql/SqlParserSetStreamingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/sql/SqlParserSetStreamingSelfTest.java
new file mode 100644
index 0000000..65bb599
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/sql/SqlParserSetStreamingSelfTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.sql;
+
+import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.sql.command.SqlSetStreamingCommand;
+
+/**
+ * Tests for SQL parser: SET STREAMING.
+ */
+public class SqlParserSetStreamingSelfTest extends SqlParserAbstractSelfTest {
+ /**
+ *
+ */
+ public void testParseSetStreaming() {
+ parseValidate("set streaming on", true, false, 2048, 0, 0, 0);
+ parseValidate("set streaming 1", true, false, 2048, 0, 0, 0);
+ parseValidate("set streaming off", false, false, 2048, 0, 0, 0);
+ parseValidate("set streaming 0", false, false, 2048, 0, 0, 0);
+ parseValidate("set streaming on batch_size 100", true, false, 100, 0, 0, 0);
+ parseValidate("set streaming on flush_frequency 500", true, false, 2048, 0, 0, 500);
+ parseValidate("set streaming on per_node_buffer_size 100", true, false, 2048, 0, 100, 0);
+ parseValidate("set streaming on per_node_parallel_operations 4", true, false, 2048, 4, 0, 0);
+ parseValidate("set streaming on allow_overwrite on", true, true, 2048, 0, 0, 0);
+ parseValidate("set streaming on allow_overwrite off", true, false, 2048, 0, 0, 0);
+ parseValidate("set streaming on per_node_buffer_size 50 flush_frequency 500 " +
+ "per_node_parallel_operations 4 allow_overwrite on batch_size 100", true, true, 100, 4, 50, 500);
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set",
+ "Failed to parse SQL statement \"set[*]\": Unexpected end of command (expected: \"STREAMING\")");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming",
+ "Failed to parse SQL statement \"set streaming[*]\": Unexpected end of command (expected: " +
+ "\"ON\", \"OFF\", \"1\", \"0\")");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming table",
+ "Failed to parse SQL statement \"set streaming [*]table\": Unexpected token: \"TABLE\" (expected: " +
+ "\"ON\", \"OFF\", \"1\", \"0\")");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming -1",
+ "Failed to parse SQL statement \"set streaming [*]-1\": Unexpected token: \"-\" (expected: " +
+ "\"ON\", \"OFF\", \"1\", \"0\")");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming 500",
+ "Failed to parse SQL statement \"set streaming [*]500\": Unexpected token: \"500\" (expected: " +
+ "\"ON\", \"OFF\", \"1\", \"0\")");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming on allow_overwrite",
+ "set streaming on allow_overwrite[*]\": Unexpected end of command (expected: \"ON\", \"OFF\", \"1\", " +
+ "\"0\")");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming 1 batch_size",
+ "Failed to parse SQL statement \"set streaming 1 batch_size[*]\": Unexpected end of command " +
+ "(expected: \"[integer]\")");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming on per_node_parallel_operations -4",
+ "Failed to parse SQL statement \"set streaming on per_node_parallel_operations -[*]4\": " +
+ "Invalid per node parallel operations number (must be positive)");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming on per_node_buffer_size -4",
+ "Failed to parse SQL statement \"set streaming on per_node_buffer_size -[*]4\": " +
+ "Invalid per node buffer size (must be positive)");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming on flush_frequency -4",
+ "Failed to parse SQL statement \"set streaming on flush_frequency -[*]4\": " +
+ "Invalid flush frequency (must be positive)");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming off allow_overwrite",
+ "Failed to parse SQL statement \"set streaming off [*]allow_overwrite\": Unexpected token: " +
+ "\"ALLOW_OVERWRITE\"");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming off batch_size",
+ "Failed to parse SQL statement \"set streaming off [*]batch_size\": Unexpected token: " +
+ "\"BATCH_SIZE\"");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming off flush_frequency",
+ "Failed to parse SQL statement \"set streaming off [*]flush_frequency\": Unexpected token: " +
+ "\"FLUSH_FREQUENCY\"");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming off per_node_buffer_size",
+ "Failed to parse SQL statement \"set streaming off [*]per_node_buffer_size\": Unexpected token: " +
+ "\"PER_NODE_BUFFER_SIZE\"");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming off per_node_parallel_operations",
+ "Failed to parse SQL statement \"set streaming off [*]per_node_parallel_operations\": Unexpected token: " +
+ "\"PER_NODE_PARALLEL_OPERATIONS\"");
+
+ assertParseError(QueryUtils.DFLT_SCHEMA, "set streaming off table",
+ "Failed to parse SQL statement \"set streaming off [*]table\": Unexpected token: \"TABLE\"");
+ }
+
+ /**
+ * Parse and validate SQL script.
+ *
+ * @param sql SQL.
+ * @param expOn Expected on/off value.
+ * @param expAllowOverwrite Expected allow overwrite flag.
+ * @param expBatchSize Expected batch size.
+ * @param expParOps Expected per-node parallael operations.
+ * @param expBufSize Expected per node buffer size.
+ * @param expFlushFreq Expected flush frequency.
+ */
+ private static void parseValidate(String sql, boolean expOn, boolean expAllowOverwrite, int expBatchSize,
+ int expParOps, int expBufSize, long expFlushFreq) {
+ SqlSetStreamingCommand cmd = (SqlSetStreamingCommand)new SqlParser(QueryUtils.DFLT_SCHEMA, sql).nextCommand();
+
+ assertEquals(expOn, cmd.isTurnOn());
+
+ assertEquals(expAllowOverwrite, cmd.allowOverwrite());
+
+ assertEquals(expBatchSize, cmd.batchSize());
+
+ assertEquals(expParOps, cmd.perNodeParallelOperations());
+
+ assertEquals(expBufSize, cmd.perNodeBufferSize());
+
+ assertEquals(expFlushFreq, cmd.flushFrequency());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index b114828..8d2a820 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -299,7 +299,7 @@ public final class GridTestUtils {
call.call();
}
catch (Throwable e) {
- if (cls != e.getClass()) {
+ if (cls != e.getClass() && !cls.isAssignableFrom(e.getClass())) {
if (e.getClass() == CacheException.class && e.getCause() != null && e.getCause().getClass() == cls)
e = e.getCause();
else {
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
index ea6c7c1..62dbd50 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
@@ -43,10 +43,10 @@ import org.apache.ignite.cache.query.BulkLoadContextCursor;
import org.apache.ignite.cache.query.FieldsQueryCursor;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.processors.bulkload.BulkLoadCacheWriter;
-import org.apache.ignite.internal.processors.bulkload.BulkLoadProcessor;
import org.apache.ignite.internal.processors.bulkload.BulkLoadAckClientParameters;
+import org.apache.ignite.internal.processors.bulkload.BulkLoadCacheWriter;
import org.apache.ignite.internal.processors.bulkload.BulkLoadParser;
+import org.apache.ignite.internal.processors.bulkload.BulkLoadProcessor;
import org.apache.ignite.internal.processors.bulkload.BulkLoadStreamerWriter;
import org.apache.ignite.internal.processors.cache.CacheOperationContext;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -385,6 +385,7 @@ public class DmlStatementsProcessor {
/**
* Perform given statement against given data streamer. Only rows based INSERT is supported.
*
+ * @param schemaName Schema name.
* @param streamer Streamer to feed data to.
* @param stmt Statement.
* @param args Statement arguments.
@@ -392,81 +393,74 @@ public class DmlStatementsProcessor {
* @throws IgniteCheckedException if failed.
*/
@SuppressWarnings({"unchecked", "ConstantConditions"})
- long streamUpdateQuery(IgniteDataStreamer streamer, PreparedStatement stmt, final Object[] args)
+ long streamUpdateQuery(String schemaName, IgniteDataStreamer streamer, PreparedStatement stmt, final Object[] args)
throws IgniteCheckedException {
+ idx.checkStatementStreamable(stmt);
+
Prepared p = GridSqlQueryParser.prepared(stmt);
assert p != null;
- final UpdatePlan plan = UpdatePlanBuilder.planForStatement(p, true, idx, null, null, null);
-
- if (!F.eq(streamer.cacheName(), plan.cacheContext().name()))
- throw new IgniteSQLException("Cross cache streaming is not supported, please specify cache explicitly" +
- " in connection options", IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
-
- if (plan.mode() == UpdateMode.INSERT && plan.rowCount() > 0) {
- assert plan.isLocalSubquery();
+ final UpdatePlan plan = getPlanForStatement(schemaName, null, p, null, true, null);
- final GridCacheContext cctx = plan.cacheContext();
+ assert plan.isLocalSubquery();
- QueryCursorImpl<List<?>> cur;
+ final GridCacheContext cctx = plan.cacheContext();
- final ArrayList<List<?>> data = new ArrayList<>(plan.rowCount());
+ QueryCursorImpl<List<?>> cur;
- QueryCursorImpl<List<?>> stepCur = new QueryCursorImpl<>(new Iterable<List<?>>() {
- @Override public Iterator<List<?>> iterator() {
- try {
- Iterator<List<?>> it;
+ final ArrayList<List<?>> data = new ArrayList<>(plan.rowCount());
- if (!F.isEmpty(plan.selectQuery())) {
- GridQueryFieldsResult res = idx.queryLocalSqlFields(idx.schema(cctx.name()),
- plan.selectQuery(), F.asList(U.firstNotNull(args, X.EMPTY_OBJECT_ARRAY)),
- null, false, 0, null);
+ QueryCursorImpl<List<?>> stepCur = new QueryCursorImpl<>(new Iterable<List<?>>() {
+ @Override public Iterator<List<?>> iterator() {
+ try {
+ Iterator<List<?>> it;
- it = res.iterator();
- }
- else
- it = plan.createRows(U.firstNotNull(args, X.EMPTY_OBJECT_ARRAY)).iterator();
+ if (!F.isEmpty(plan.selectQuery())) {
+ GridQueryFieldsResult res = idx.queryLocalSqlFields(idx.schema(cctx.name()),
+ plan.selectQuery(), F.asList(U.firstNotNull(args, X.EMPTY_OBJECT_ARRAY)),
+ null, false, 0, null);
- return new GridQueryCacheObjectsIterator(it, idx.objectContext(), cctx.keepBinary());
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
+ it = res.iterator();
}
- }
- }, null);
-
- data.addAll(stepCur.getAll());
+ else
+ it = plan.createRows(U.firstNotNull(args, X.EMPTY_OBJECT_ARRAY)).iterator();
- cur = new QueryCursorImpl<>(new Iterable<List<?>>() {
- @Override public Iterator<List<?>> iterator() {
- return data.iterator();
+ return new GridQueryCacheObjectsIterator(it, idx.objectContext(), cctx.keepBinary());
}
- }, null);
-
- if (plan.rowCount() == 1) {
- IgniteBiTuple t = plan.processRow(cur.iterator().next());
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+ }, null);
- streamer.addData(t.getKey(), t.getValue());
+ data.addAll(stepCur.getAll());
- return 1;
+ cur = new QueryCursorImpl<>(new Iterable<List<?>>() {
+ @Override public Iterator<List<?>> iterator() {
+ return data.iterator();
}
+ }, null);
- Map<Object, Object> rows = new LinkedHashMap<>(plan.rowCount());
+ if (plan.rowCount() == 1) {
+ IgniteBiTuple t = plan.processRow(cur.iterator().next());
- for (List<?> row : cur) {
- final IgniteBiTuple t = plan.processRow(row);
+ streamer.addData(t.getKey(), t.getValue());
- rows.put(t.getKey(), t.getValue());
- }
+ return 1;
+ }
+
+ Map<Object, Object> rows = new LinkedHashMap<>(plan.rowCount());
- streamer.addData(rows);
+ for (List<?> row : cur) {
+ final IgniteBiTuple t = plan.processRow(row);
- return rows.size();
+ rows.put(t.getKey(), t.getValue());
}
- else
- throw new IgniteSQLException("Only tuple based INSERT statements are supported in streaming mode",
- IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+
+ streamer.addData(rows);
+
+ return rows.size();
}
/**
@@ -519,7 +513,7 @@ public class DmlStatementsProcessor {
.setPageSize(fieldsQry.getPageSize())
.setTimeout(fieldsQry.getTimeout(), TimeUnit.MILLISECONDS);
- cur = (QueryCursorImpl<List<?>>)idx.querySqlFields(schemaName, newFieldsQry, true, true,
+ cur = (QueryCursorImpl<List<?>>)idx.querySqlFields(schemaName, newFieldsQry, null, true, true,
cancel).get(0);
}
else if (plan.hasRows())
@@ -610,7 +604,7 @@ public class DmlStatementsProcessor {
* @return Update plan.
*/
@SuppressWarnings({"unchecked", "ConstantConditions"})
- private UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared p, SqlFieldsQuery fieldsQry,
+ UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared p, SqlFieldsQuery fieldsQry,
boolean loc, @Nullable Integer errKeysPos) throws IgniteCheckedException {
H2CachedStatementKey planKey = H2CachedStatementKey.forDmlStatement(schema, p.getSQL(), fieldsQry, loc);
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java
index cfbb7bb..e9d9f90 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java
@@ -17,8 +17,16 @@
package org.apache.ignite.internal.processors.query.h2;
+import java.lang.reflect.Constructor;
+import java.sql.Connection;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
@@ -33,12 +41,7 @@ import org.h2.table.IndexColumn;
import org.h2.value.DataType;
import org.h2.value.Value;
-import java.lang.reflect.Constructor;
-import java.sql.Connection;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
+import static org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.UPDATE_RESULT_META;
/**
* H2 utility methods.
@@ -267,4 +270,17 @@ public class H2Utils {
private H2Utils() {
// No-op.
}
+
+ /**
+ * @return Single-column, single-row cursor with 0 as number of updated records.
+ */
+ @SuppressWarnings("unchecked")
+ public static QueryCursorImpl<List<?>> zeroCursor() {
+ QueryCursorImpl<List<?>> resCur = (QueryCursorImpl<List<?>>)new QueryCursorImpl(Collections.singletonList
+ (Collections.singletonList(0L)), null, false);
+
+ resCur.fieldsMeta(UPDATE_RESULT_META);
+
+ return resCur;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 361623c..65f08b2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -92,6 +92,7 @@ import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.QueryField;
import org.apache.ignite.internal.processors.query.QueryIndexDescriptorImpl;
import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.processors.query.SqlClientContext;
import org.apache.ignite.internal.processors.query.h2.database.H2RowFactory;
import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex;
import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasInnerIO;
@@ -100,6 +101,7 @@ import org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO;
import org.apache.ignite.internal.processors.query.h2.database.io.H2LeafIO;
import org.apache.ignite.internal.processors.query.h2.ddl.DdlStatementsProcessor;
import org.apache.ignite.internal.processors.query.h2.dml.DmlUtils;
+import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlan;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2DefaultTableEngine;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2PlainRowFactory;
@@ -120,11 +122,12 @@ import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisito
import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.sql.SqlParseException;
import org.apache.ignite.internal.sql.SqlParser;
-import org.apache.ignite.internal.sql.command.SqlBulkLoadCommand;
import org.apache.ignite.internal.sql.command.SqlAlterTableCommand;
+import org.apache.ignite.internal.sql.command.SqlBulkLoadCommand;
import org.apache.ignite.internal.sql.command.SqlCommand;
import org.apache.ignite.internal.sql.command.SqlCreateIndexCommand;
import org.apache.ignite.internal.sql.command.SqlDropIndexCommand;
+import org.apache.ignite.internal.sql.command.SqlSetStreamingCommand;
import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap;
import org.apache.ignite.internal.util.GridEmptyCloseableIterator;
import org.apache.ignite.internal.util.GridSpinBusyLock;
@@ -149,7 +152,6 @@ import org.apache.ignite.spi.indexing.IndexingQueryFilterImpl;
import org.h2.api.ErrorCode;
import org.h2.api.JavaObjectSerializer;
import org.h2.command.Prepared;
-import org.h2.command.dml.Insert;
import org.h2.command.dml.NoOperation;
import org.h2.engine.Session;
import org.h2.engine.SysProperties;
@@ -191,7 +193,7 @@ import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType
@SuppressWarnings({"UnnecessaryFullyQualifiedName", "NonFinalStaticVariableUsedInClassInitialization"})
public class IgniteH2Indexing implements GridQueryIndexing {
public static final Pattern INTERNAL_CMD_RE = Pattern.compile(
- "^(create|drop)\\s+index|^alter\\s+table|^copy", Pattern.CASE_INSENSITIVE);
+ "^(create|drop)\\s+index|^alter\\s+table|^copy|^set", Pattern.CASE_INSENSITIVE);
/*
* Register IO for indexes.
@@ -500,10 +502,10 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
/** {@inheritDoc} */
- @Override public PreparedStatement prepareNativeStatement(String schemaName, String sql) throws SQLException {
+ @Override public PreparedStatement prepareNativeStatement(String schemaName, String sql) {
Connection conn = connectionForSchema(schemaName);
- return prepareStatement(conn, sql, true);
+ return prepareStatementAndCaches(conn, sql);
}
/**
@@ -1013,7 +1015,55 @@ public class IgniteH2Indexing implements GridQueryIndexing {
throw new IgniteSQLException(e);
}
- return dmlProc.streamUpdateQuery(streamer, stmt, params);
+ return dmlProc.streamUpdateQuery(schemaName, streamer, stmt, params);
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("ForLoopReplaceableByForEach")
+ @Override public List<Long> streamBatchedUpdateQuery(String schemaName, String qry, List<Object[]> params,
+ SqlClientContext cliCtx) throws IgniteCheckedException {
+ if (cliCtx == null || !cliCtx.isStream()) {
+ U.warn(log, "Connection is not in streaming mode.");
+
+ return zeroBatchedStreamedUpdateResult(params.size());
+ }
+
+ final Connection conn = connectionForSchema(schemaName);
+
+ final PreparedStatement stmt = prepareStatementAndCaches(conn, qry);
+
+ if (GridSqlQueryParser.checkMultipleStatements(stmt))
+ throw new IgniteSQLException("Multiple statements queries are not supported for streaming mode.",
+ IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+
+ checkStatementStreamable(stmt);
+
+ Prepared p = GridSqlQueryParser.prepared(stmt);
+
+ UpdatePlan plan = dmlProc.getPlanForStatement(schemaName, conn, p, null, true, null);
+
+ IgniteDataStreamer<?, ?> streamer = cliCtx.streamerForCache(plan.cacheContext().name());
+
+ assert streamer != null;
+
+ List<Long> res = new ArrayList<>(params.size());
+
+ for (int i = 0; i < params.size(); i++)
+ res.add(dmlProc.streamUpdateQuery(schemaName, streamer, stmt, params.get(i)));
+
+ return res;
+ }
+
+ /**
+ * @param size Result size.
+ * @return List of given size filled with 0Ls.
+ */
+ private static List<Long> zeroBatchedStreamedUpdateResult(int size) {
+ Long[] res = new Long[size];
+
+ Arrays.fill(res, 0);
+
+ return Arrays.asList(res);
}
/**
@@ -1399,7 +1449,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
fqry.setTimeout(qry.getTimeout(), TimeUnit.MILLISECONDS);
final QueryCursor<List<?>> res =
- querySqlFields(schemaName, fqry, keepBinary, true, null).get(0);
+ querySqlFields(schemaName, fqry, null, keepBinary, true, null).get(0);
final Iterable<Cache.Entry<K, V>> converted = new Iterable<Cache.Entry<K, V>>() {
@Override public Iterator<Cache.Entry<K, V>> iterator() {
@@ -1435,19 +1485,21 @@ public class IgniteH2Indexing implements GridQueryIndexing {
* Try executing query using native facilities.
*
* @param schemaName Schema name.
- * @param qry Query.
+ * @param sql Query.
+ * @param cliCtx Client context, or {@code null} if not applicable.
* @return Result or {@code null} if cannot parse/process this query.
*/
- private List<FieldsQueryCursor<List<?>>> tryQueryDistributedSqlFieldsNative(String schemaName, SqlFieldsQuery qry) {
+ private List<FieldsQueryCursor<List<?>>> tryQueryDistributedSqlFieldsNative(String schemaName, String sql,
+ @Nullable SqlClientContext cliCtx) {
// Heuristic check for fast return.
- if (!INTERNAL_CMD_RE.matcher(qry.getSql().trim()).find())
+ if (!INTERNAL_CMD_RE.matcher(sql.trim()).find())
return null;
// Parse.
SqlCommand cmd;
try {
- SqlParser parser = new SqlParser(schemaName, qry.getSql());
+ SqlParser parser = new SqlParser(schemaName, sql);
cmd = parser.nextCommand();
@@ -1455,15 +1507,20 @@ public class IgniteH2Indexing implements GridQueryIndexing {
if (parser.nextCommand() != null)
return null;
- // Currently supported commands are: CREATE/DROP INDEX/COPY/ALTER TABLE
+ // Currently supported commands are:
+ // CREATE/DROP INDEX
+ // COPY
+ // ALTER TABLE
+ // SET STREAMING
if (!(cmd instanceof SqlCreateIndexCommand || cmd instanceof SqlDropIndexCommand ||
- cmd instanceof SqlBulkLoadCommand || cmd instanceof SqlAlterTableCommand))
+ cmd instanceof SqlAlterTableCommand || cmd instanceof SqlBulkLoadCommand ||
+ cmd instanceof SqlSetStreamingCommand))
return null;
}
catch (Exception e) {
// Cannot parse, return.
if (log.isDebugEnabled())
- log.debug("Failed to parse SQL with native parser [qry=" + qry.getSql() + ", err=" + e + ']');
+ log.debug("Failed to parse SQL with native parser [qry=" + sql + ", err=" + e + ']');
if (!IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_SQL_PARSER_DISABLE_H2_FALLBACK))
return null;
@@ -1473,24 +1530,40 @@ public class IgniteH2Indexing implements GridQueryIndexing {
if (e instanceof SqlParseException)
code = ((SqlParseException)e).code();
- throw new IgniteSQLException("Failed to parse DDL statement: " + qry.getSql() + ": " + e.getMessage(),
+ throw new IgniteSQLException("Failed to parse DDL statement: " + sql + ": " + e.getMessage(),
code, e);
}
// Execute.
if (cmd instanceof SqlBulkLoadCommand) {
- FieldsQueryCursor<List<?>> cursor = dmlProc.runNativeDmlStatement(qry.getSql(), cmd);
+ FieldsQueryCursor<List<?>> cursor = dmlProc.runNativeDmlStatement(sql, cmd);
return Collections.singletonList(cursor);
}
+ else if (cmd instanceof SqlSetStreamingCommand) {
+ if (cliCtx == null)
+ throw new IgniteSQLException("SET STREAMING command can only be executed from JDBC or ODBC driver.");
+
+ SqlSetStreamingCommand setCmd = (SqlSetStreamingCommand)cmd;
+
+ boolean on = setCmd.isTurnOn();
+
+ if (on)
+ cliCtx.enableStreaming(setCmd.allowOverwrite(), setCmd.flushFrequency(),
+ setCmd.perNodeBufferSize(), setCmd.perNodeParallelOperations());
+ else
+ cliCtx.disableStreaming();
+
+ return Collections.singletonList(H2Utils.zeroCursor());
+ }
else {
try {
- FieldsQueryCursor<List<?>> cursor = ddlProc.runDdlStatement(qry.getSql(), cmd);
+ FieldsQueryCursor<List<?>> cursor = ddlProc.runDdlStatement(sql, cmd);
return Collections.singletonList(cursor);
}
catch (IgniteCheckedException e) {
- throw new IgniteSQLException("Failed to execute DDL statement [stmt=" + qry.getSql() + "]: "
+ throw new IgniteSQLException("Failed to execute DDL statement [stmt=" + sql + "]: "
+ e.getMessage(), e);
}
}
@@ -1512,10 +1585,10 @@ public class IgniteH2Indexing implements GridQueryIndexing {
/** {@inheritDoc} */
- @SuppressWarnings("StringEquality")
+ @SuppressWarnings({"StringEquality", "unchecked"})
@Override public List<FieldsQueryCursor<List<?>>> querySqlFields(String schemaName, SqlFieldsQuery qry,
- boolean keepBinary, boolean failOnMultipleStmts, GridQueryCancel cancel) {
- List<FieldsQueryCursor<List<?>>> res = tryQueryDistributedSqlFieldsNative(schemaName, qry);
+ @Nullable SqlClientContext cliCtx, boolean keepBinary, boolean failOnMultipleStmts, GridQueryCancel cancel) {
+ List<FieldsQueryCursor<List<?>>> res = tryQueryDistributedSqlFieldsNative(schemaName, qry.getSql(), cliCtx);
if (res != null)
return res;
@@ -1553,8 +1626,8 @@ public class IgniteH2Indexing implements GridQueryIndexing {
// We may use this cached statement only for local queries and non queries.
if (qry.isLocal() || !prepared.isQuery())
- return (List<FieldsQueryCursor<List<?>>>)doRunPrepared(schemaName, prepared, qry, null, null,
- keepBinary, cancel);
+ return (List<FieldsQueryCursor<List<?>>>)doRunPrepared(schemaName, prepared, qry, null,
+ null, keepBinary, cancel);
}
}
@@ -1602,12 +1675,11 @@ public class IgniteH2Indexing implements GridQueryIndexing {
* @param twoStepQry Two-step query if this query must be executed in a distributed way.
* @param meta Metadata for {@code twoStepQry}.
* @param keepBinary Whether binary objects must not be deserialized automatically.
- * @param cancel Query cancel state holder.
- * @return Query result.
+ * @param cancel Query cancel state holder. @return Query result.
*/
private List<? extends FieldsQueryCursor<List<?>>> doRunPrepared(String schemaName, Prepared prepared,
- SqlFieldsQuery qry, GridCacheTwoStepQuery twoStepQry, List<GridQueryFieldMetadata> meta, boolean keepBinary,
- GridQueryCancel cancel) {
+ SqlFieldsQuery qry, GridCacheTwoStepQuery twoStepQry,
+ List<GridQueryFieldMetadata> meta, boolean keepBinary, GridQueryCancel cancel) {
String sqlQry = qry.getSql();
boolean loc = qry.isLocal();
@@ -2276,10 +2348,10 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
/** {@inheritDoc} */
- @Override public boolean isInsertStatement(PreparedStatement nativeStmt) {
- Prepared prep = GridSqlQueryParser.prepared(nativeStmt);
-
- return prep instanceof Insert;
+ @Override public void checkStatementStreamable(PreparedStatement nativeStmt) {
+ if (!GridSqlQueryParser.isStreamableInsertStatement(nativeStmt))
+ throw new IgniteSQLException("Streaming mode supports only INSERT commands without subqueries.",
+ IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
index 6f5b51f..d4f2b0e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
@@ -43,6 +43,7 @@ import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.QueryEntityEx;
import org.apache.ignite.internal.processors.query.QueryField;
import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.processors.query.h2.H2Utils;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlterTableAddColumn;
@@ -106,7 +107,7 @@ public class DdlStatementsProcessor {
* @throws IgniteCheckedException On error.
*/
@SuppressWarnings("unchecked")
- public FieldsQueryCursor<List<?>> runDdlStatement(String sql, SqlCommand cmd) throws IgniteCheckedException{
+ public FieldsQueryCursor<List<?>> runDdlStatement(String sql, SqlCommand cmd) throws IgniteCheckedException {
IgniteInternalFuture fut;
try {
@@ -211,12 +212,7 @@ public class DdlStatementsProcessor {
if (fut != null)
fut.get();
- QueryCursorImpl<List<?>> resCur = (QueryCursorImpl<List<?>>)new QueryCursorImpl(Collections.singletonList
- (Collections.singletonList(0L)), null, false);
-
- resCur.fieldsMeta(UPDATE_RESULT_META);
-
- return resCur;
+ return H2Utils.zeroCursor();
}
catch (SchemaOperationException e) {
throw convert(e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
index 10d485a..98fbb97 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
@@ -505,7 +505,7 @@ public final class UpdatePlan {
/**
* @return Local subquery flag.
*/
- @Nullable public boolean isLocalSubquery() {
+ public boolean isLocalSubquery() {
return isLocSubqry;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
index bced836..d897ac7 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
@@ -85,20 +85,21 @@ public final class UpdatePlanBuilder {
* @param loc Local query flag.
* @param idx Indexing.
* @param conn Connection.
- * @param fieldsQuery Original query.
+ * @param fieldsQry Original query.
* @return Update plan.
*/
public static UpdatePlan planForStatement(Prepared prepared, boolean loc, IgniteH2Indexing idx,
- @Nullable Connection conn, @Nullable SqlFieldsQuery fieldsQuery, @Nullable Integer errKeysPos)
+ @Nullable Connection conn, @Nullable SqlFieldsQuery fieldsQry, @Nullable Integer errKeysPos)
throws IgniteCheckedException {
- assert !prepared.isQuery();
-
GridSqlStatement stmt = new GridSqlQueryParser(false).parse(prepared);
if (stmt instanceof GridSqlMerge || stmt instanceof GridSqlInsert)
- return planForInsert(stmt, loc, idx, conn, fieldsQuery);
+ return planForInsert(stmt, loc, idx, conn, fieldsQry);
+ else if (stmt instanceof GridSqlUpdate || stmt instanceof GridSqlDelete)
+ return planForUpdate(stmt, loc, idx, conn, fieldsQry, errKeysPos);
else
- return planForUpdate(stmt, loc, idx, conn, fieldsQuery, errKeysPos);
+ throw new IgniteSQLException("Unsupported operation: " + prepared.getSQL(),
+ IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
index 04bc212..2d2c25c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
@@ -1993,6 +1993,18 @@ public class GridSqlQueryParser {
}
/**
+ * Check if passed statement is insert statement eligible for streaming.
+ *
+ * @param nativeStmt Native statement.
+ * @return {@code True} if streamable insert.
+ */
+ public static boolean isStreamableInsertStatement(PreparedStatement nativeStmt) {
+ Prepared prep = prepared(nativeStmt);
+
+ return prep instanceof Insert && INSERT_QUERY.get((Insert)prep) == null;
+ }
+
+ /**
* @param cond Condition.
* @param o Object.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
index 069bdd7..cf8bb2e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
@@ -140,7 +140,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
SqlFieldsQuery qry = new SqlFieldsQuery("select f.productId, p.name, f.price " +
"from FactPurchase f, \"replicated-prod\".DimProduct p where p.id = f.productId ");
- for (List<?> o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
+ for (List<?> o : qryProc.querySqlFields(cache.context(), qry, null, false, true).get(0).getAll()) {
X.println("___ -> " + o);
set1.add((Integer)o.get(0));
@@ -154,7 +154,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
qry = new SqlFieldsQuery("select productId from FactPurchase group by productId");
- for (List<?> o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
+ for (List<?> o : qryProc.querySqlFields(cache.context(), qry, null, false, true).get(0).getAll()) {
X.println("___ -> " + o);
assertTrue(set0.add((Integer) o.get(0)));
@@ -173,7 +173,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
"where p.id = f.productId " +
"group by f.productId, p.name");
- for (List<?> o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
+ for (List<?> o : qryProc.querySqlFields(cache.context(), qry, null, false, true).get(0).getAll()) {
X.println("___ -> " + o);
assertTrue(names.add((String)o.get(0)));
@@ -190,7 +190,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
"group by f.productId, p.name " +
"having s >= 15");
- for (List<?> o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
+ for (List<?> o : qryProc.querySqlFields(cache.context(), qry, null, false, true).get(0).getAll()) {
X.println("___ -> " + o);
assertTrue(i(o, 1) >= 15);
@@ -203,7 +203,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
qry = new SqlFieldsQuery("select top 3 distinct productId " +
"from FactPurchase f order by productId desc ");
- for (List<?> o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
+ for (List<?> o : qryProc.querySqlFields(cache.context(), qry, null, false, true).get(0).getAll()) {
X.println("___ -> " + o);
assertEquals(top--, o.get(0));
@@ -216,7 +216,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
qry = new SqlFieldsQuery("select distinct productId " +
"from FactPurchase f order by productId desc limit 2 offset 1");
- for (List<?> o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
+ for (List<?> o : qryProc.querySqlFields(cache.context(), qry, null, false, true).get(0).getAll()) {
X.println("___ -> " + o);
assertEquals(top--, o.get(0));
@@ -256,13 +256,13 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
GridTestUtils.assertThrows(log,
new Callable<Object>() {
@Override public Object call() throws Exception {
- qryProc.querySqlFields(cache.context(), qry, false, true);
+ qryProc.querySqlFields(cache.context(), qry, null, false, true);
return null;
}
}, IgniteSQLException.class, "Multiple statements queries are not supported");
- List<FieldsQueryCursor<List<?>>> cursors = qryProc.querySqlFields(cache.context(), qry, false, false);
+ List<FieldsQueryCursor<List<?>>> cursors = qryProc.querySqlFields(cache.context(), qry, null, false, false);
assertEquals(2, cursors.size());
@@ -274,7 +274,7 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
GridTestUtils.assertThrows(log,
new Callable<Object>() {
@Override public Object call() throws Exception {
- qryProc.querySqlFields(cache.context(), qry, false, false);
+ qryProc.querySqlFields(cache.context(), qry, null, false, false);
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index f004453..c01eaa6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -170,6 +170,7 @@ import org.apache.ignite.internal.processors.sql.SqlConnectorConfigurationValida
import org.apache.ignite.internal.sql.SqlParserBulkLoadSelfTest;
import org.apache.ignite.internal.sql.SqlParserCreateIndexSelfTest;
import org.apache.ignite.internal.sql.SqlParserDropIndexSelfTest;
+import org.apache.ignite.internal.sql.SqlParserSetStreamingSelfTest;
import org.apache.ignite.spi.communication.tcp.GridOrderedMessageCancelSelfTest;
import org.apache.ignite.testframework.IgniteTestSuite;
@@ -187,6 +188,7 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
suite.addTestSuite(SqlParserCreateIndexSelfTest.class);
suite.addTestSuite(SqlParserDropIndexSelfTest.class);
suite.addTestSuite(SqlParserBulkLoadSelfTest.class);
+ suite.addTestSuite(SqlParserSetStreamingSelfTest.class);
suite.addTestSuite(SqlConnectorConfigurationValidationSelfTest.class);
suite.addTestSuite(ClientConnectorConfigurationValidationSelfTest.class);
[50/50] [abbrv] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-zk
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-zk
# Conflicts:
# modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
# modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
# modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
# modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
# modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/00917728
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/00917728
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/00917728
Branch: refs/heads/ignite-zk
Commit: 00917728b527abe98a5b9e582d3bf749bb8e2beb
Parents: 23ac316
Author: sboikov <sb...@gridgain.com>
Authored: Mon Mar 19 09:26:05 2018 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Mar 19 09:26:05 2018 +0300
----------------------------------------------------------------------
.../internal/managers/communication/GridIoMessageFactory.java | 4 ++--
.../ignite/internal/processors/cache/WalStateFinishMessage.java | 5 +++++
.../internal/processors/cache/WalStateProposeMessage.java | 5 +++++
.../processors/cluster/ClusterMetricsUpdateMessage.java | 2 +-
.../continuous/ContinuousRoutineStartResultMessage.java | 2 +-
.../processors/cache/IgniteClusterActivateDeactivateTest.java | 2 +-
6 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/00917728/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 7ee55f9..d7fc31f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -893,12 +893,12 @@ public class GridIoMessageFactory implements MessageFactory {
break;
- case 129:
+ case 130:
msg = new ClusterMetricsUpdateMessage();
break;
- case 130:
+ case 131:
msg = new ContinuousRoutineStartResultMessage();
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/00917728/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
index 57f25d0..4afa403 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
@@ -67,6 +67,11 @@ public class WalStateFinishMessage extends WalStateAbstractMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return S.toString(WalStateFinishMessage.class, this, "super", super.toString());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00917728/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
index 747fd6a..b9d96fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
@@ -98,6 +98,11 @@ public class WalStateProposeMessage extends WalStateAbstractMessage {
}
/** {@inheritDoc} */
+ @Override public boolean stopProcess() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return S.toString(WalStateProposeMessage.class, this, "super", super.toString());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/00917728/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
index 239dad5..90a5bd9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
@@ -138,7 +138,7 @@ public class ClusterMetricsUpdateMessage implements Message {
/** {@inheritDoc} */
@Override public short directType() {
- return 129;
+ return 130;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/00917728/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
index 9f40128..728380f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
@@ -186,7 +186,7 @@ public class ContinuousRoutineStartResultMessage implements Message {
/** {@inheritDoc} */
@Override public short directType() {
- return 130;
+ return 131;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/00917728/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 92577ad..838e56d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -1128,7 +1128,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
* @throws Exception If failed.
*/
private void stateChangeFailover3(boolean activate) throws Exception {
- testDiscoSpi = true;
+ testReconnectSpi = true;
startNodesAndBlockStatusChange(4, 0, 0, !activate);
[49/50] [abbrv] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-zk
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-zk
# Conflicts:
# modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
# modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
# modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
# modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
# modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/23ac3166
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/23ac3166
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/23ac3166
Branch: refs/heads/ignite-zk
Commit: 23ac31662375b2e314dc586830908331c58306ea
Parents: a65d955 98ba7b3
Author: sboikov <sb...@gridgain.com>
Authored: Mon Mar 19 09:21:34 2018 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Mar 19 09:21:34 2018 +0300
----------------------------------------------------------------------
.gitignore | 4 +
NOTICE | 2 +-
RELEASE_NOTES.txt | 94 +
assembly/NOTICE_FABRIC | 2 +-
assembly/NOTICE_HADOOP | 2 +-
bin/control.bat | 16 +-
bin/control.sh | 15 +-
bin/ignite.bat | 24 +-
bin/ignite.sh | 15 +-
bin/ignitevisorcmd.bat | 13 +-
bin/ignitevisorcmd.sh | 12 +
bin/include/build-classpath.bat | 13 +-
bin/include/build-classpath.sh | 7 +-
bin/include/functions.sh | 13 +-
bin/include/parseargs.bat | 5 +-
bin/include/setenv.bat | 8 +-
bin/include/setenv.sh | 5 +
config/ignite-log4j2.xml | 8 +-
examples/README-LGPL.txt | 7 -
examples/README.txt | 17 -
examples/pom-standalone-lgpl.xml | 66 +-
examples/pom-standalone.xml | 66 +-
examples/pom.xml | 100 +-
.../examples/cluster/ClusterGroupExample.java | 86 +
.../ignite/examples/cluster/package-info.java | 22 +
.../computegrid/ComputeAsyncExample.java | 17 +-
.../computegrid/ComputeBroadcastExample.java | 44 +-
.../computegrid/ComputeCallableExample.java | 18 +-
.../computegrid/ComputeClosureExample.java | 22 +-
.../computegrid/ComputeRunnableExample.java | 10 +-
.../cluster/ClusterGroupExample.java | 99 -
.../computegrid/cluster/package-info.java | 22 -
.../examples/datagrid/CacheAffinityExample.java | 38 +-
.../examples/datagrid/CacheApiExample.java | 26 +-
.../examples/datagrid/CacheAsyncApiExample.java | 37 +-
...heContinuousQueryWithTransformerExample.java | 150 +
.../datagrid/CacheEntryProcessorExample.java | 46 +-
.../ignite/examples/datagrid/package-info.java | 2 +-
.../IgniteExecutorServiceExample.java | 8 +-
.../ignite/examples/events/EventsExample.java | 32 +-
.../examples/messaging/MessagingExample.java | 95 +-
.../messaging/MessagingPingPongExample.java | 46 +-
.../examples/ml/MLExamplesCommonArgs.java | 31 +
.../clustering/DatasetWithObviousStructure.java | 105 +
.../ml/clustering/FuzzyCMeansExample.java | 134 +
.../ml/clustering/FuzzyCMeansLocalExample.java | 95 +
.../KMeansDistributedClustererExample.java | 97 +
.../clustering/KMeansLocalClustererExample.java | 106 +
.../examples/ml/clustering/package-info.java | 22 +
.../AlgorithmSpecificDatasetExample.java | 197 +
.../ml/dataset/CacheBasedDatasetExample.java | 90 +
.../ml/dataset/LocalDatasetExample.java | 84 +
.../examples/ml/dataset/model/Person.java | 58 +
.../examples/ml/dataset/model/package-info.java | 22 +
.../examples/ml/dataset/package-info.java | 22 +
.../ignite/examples/ml/genetic/change/Coin.java | 88 +
.../OptimizeMakeChangeFitnessFunction.java | 106 +
.../change/OptimizeMakeChangeGAExample.java | 207 +
.../OptimizeMakeChangeTerminateCriteria.java | 83 +
.../helloworld/HelloWorldFitnessFunction.java | 64 +
.../genetic/helloworld/HelloWorldGAExample.java | 120 +
.../helloworld/HelloWorldTerminateCriteria.java | 88 +
.../ignite/examples/ml/genetic/movie/Movie.java | 133 +
.../ml/genetic/movie/MovieFitnessFunction.java | 105 +
.../ml/genetic/movie/MovieGAExample.java | 267 ++
.../genetic/movie/MovieTerminateCriteria.java | 88 +
.../KNNClassificationExample.java | 154 +
.../ml/knn/classification/package-info.java | 22 +
.../ignite/examples/ml/knn/package-info.java | 22 +
.../ml/knn/regression/KNNRegressionExample.java | 155 +
.../ml/knn/regression/package-info.java | 22 +
.../CholeskyDecompositionExample.java | 80 +
.../EigenDecompositionExample.java | 69 +
.../decompositions/LUDecompositionExample.java | 83 +
.../decompositions/QRDecompositionExample.java | 82 +
.../SingularValueDecompositionExample.java | 70 +
.../ml/math/decompositions/package-info.java | 22 +
.../ml/math/matrix/CacheMatrixExample.java | 102 +
.../ml/math/matrix/ExampleMatrixStorage.java | 175 +
.../math/matrix/MatrixCustomStorageExample.java | 141 +
.../examples/ml/math/matrix/MatrixExample.java | 79 +
.../ml/math/matrix/MatrixExampleUtil.java | 52 +
.../ml/math/matrix/OffHeapMatrixExample.java | 84 +
.../matrix/SparseDistributedMatrixExample.java | 70 +
.../ml/math/matrix/SparseMatrixExample.java | 84 +
.../examples/ml/math/matrix/package-info.java | 22 +
.../ignite/examples/ml/math/package-info.java | 22 +
.../examples/ml/math/tracer/TracerExample.java | 63 +
.../examples/ml/math/tracer/package-info.java | 22 +
.../ml/math/vector/CacheVectorExample.java | 109 +
.../ml/math/vector/ExampleVectorStorage.java | 127 +
.../ml/math/vector/OffHeapVectorExample.java | 78 +
.../ml/math/vector/SparseVectorExample.java | 76 +
.../math/vector/VectorCustomStorageExample.java | 120 +
.../examples/ml/math/vector/VectorExample.java | 73 +
.../examples/ml/math/vector/package-info.java | 22 +
.../examples/ml/nn/MLPGroupTrainerExample.java | 140 +
.../examples/ml/nn/MLPLocalTrainerExample.java | 161 +
.../ignite/examples/ml/nn/package-info.java | 22 +
.../apache/ignite/examples/ml/package-info.java | 22 +
.../ml/preprocessing/NormalizationExample.java | 109 +
.../examples/ml/preprocessing/package-info.java | 22 +
...nWithLSQRTrainerAndNormalizationExample.java | 184 +
...dLinearRegressionWithLSQRTrainerExample.java | 170 +
...tedLinearRegressionWithQRTrainerExample.java | 136 +
...edLinearRegressionWithSGDTrainerExample.java | 137 +
.../ml/regression/linear/package-info.java | 22 +
.../examples/ml/regression/package-info.java | 22 +
.../binary/SVMBinaryClassificationExample.java | 233 ++
.../examples/ml/svm/binary/package-info.java | 22 +
.../ignite/examples/ml/svm/package-info.java | 22 +
.../examples/ml/trees/DecisionTreesExample.java | 354 ++
.../ignite/examples/ml/trees/package-info.java | 22 +
.../ignite/examples/sql/SqlDmlExample.java | 12 +-
.../ignite/examples/sql/SqlJdbcCopyExample.java | 107 +
.../streaming/StreamTransformerExample.java | 28 +-
.../streaming/StreamVisitorExample.java | 84 +-
.../ignite/examples/streaming/package-info.java | 1 +
.../java8/cluster/ClusterGroupExample.java | 86 -
.../examples/java8/cluster/package-info.java | 22 -
.../java8/computegrid/ComputeAsyncExample.java | 75 -
.../computegrid/ComputeBroadcastExample.java | 102 -
.../computegrid/ComputeCallableExample.java | 75 -
.../computegrid/ComputeClosureExample.java | 71 -
.../computegrid/ComputeRunnableExample.java | 64 -
.../java8/computegrid/package-info.java | 22 -
.../java8/datagrid/CacheAffinityExample.java | 142 -
.../java8/datagrid/CacheApiExample.java | 110 -
.../java8/datagrid/CacheAsyncApiExample.java | 90 -
.../datagrid/CacheEntryProcessorExample.java | 152 -
.../examples/java8/datagrid/package-info.java | 22 -
.../IgniteExecutorServiceExample.java | 70 -
.../java8/datastructures/package-info.java | 22 -
.../examples/java8/events/EventsExample.java | 135 -
.../examples/java8/events/package-info.java | 22 -
.../java8/messaging/MessagingExample.java | 161 -
.../messaging/MessagingPingPongExample.java | 113 -
.../examples/java8/messaging/package-info.java | 22 -
.../ignite/examples/java8/package-info.java | 23 -
.../streaming/StreamTransformerExample.java | 108 -
.../java8/streaming/StreamVisitorExample.java | 177 -
.../examples/java8/streaming/package-info.java | 22 -
.../clustering/DatasetWithObviousStructure.java | 105 -
.../ml/clustering/FuzzyCMeansExample.java | 134 -
.../ml/clustering/FuzzyCMeansLocalExample.java | 95 -
.../KMeansDistributedClustererExample.java | 97 -
.../clustering/KMeansLocalClustererExample.java | 106 -
.../examples/ml/clustering/package-info.java | 22 -
.../KNNClassificationExample.java | 155 -
.../ml/knn/classification/package-info.java | 22 -
.../ignite/examples/ml/knn/datasets/README.md | 2 -
.../ml/knn/datasets/cleared_machines.txt | 209 -
.../ignite/examples/ml/knn/datasets/iris.txt | 150 -
.../ignite/examples/ml/knn/package-info.java | 22 -
.../ml/knn/regression/KNNRegressionExample.java | 157 -
.../ml/knn/regression/package-info.java | 22 -
.../CholeskyDecompositionExample.java | 80 -
.../EigenDecompositionExample.java | 69 -
.../decompositions/LUDecompositionExample.java | 83 -
.../decompositions/QRDecompositionExample.java | 82 -
.../SingularValueDecompositionExample.java | 70 -
.../ml/math/decompositions/package-info.java | 22 -
.../ml/math/matrix/CacheMatrixExample.java | 102 -
.../ml/math/matrix/ExampleMatrixStorage.java | 175 -
.../math/matrix/MatrixCustomStorageExample.java | 141 -
.../examples/ml/math/matrix/MatrixExample.java | 79 -
.../ml/math/matrix/MatrixExampleUtil.java | 52 -
.../ml/math/matrix/OffHeapMatrixExample.java | 84 -
.../matrix/SparseDistributedMatrixExample.java | 70 -
.../ml/math/matrix/SparseMatrixExample.java | 84 -
.../examples/ml/math/matrix/package-info.java | 22 -
.../ignite/examples/ml/math/package-info.java | 22 -
.../examples/ml/math/tracer/TracerExample.java | 63 -
.../examples/ml/math/tracer/package-info.java | 22 -
.../ml/math/vector/CacheVectorExample.java | 109 -
.../ml/math/vector/ExampleVectorStorage.java | 127 -
.../ml/math/vector/OffHeapVectorExample.java | 78 -
.../ml/math/vector/SparseVectorExample.java | 76 -
.../math/vector/VectorCustomStorageExample.java | 120 -
.../examples/ml/math/vector/VectorExample.java | 73 -
.../examples/ml/math/vector/package-info.java | 22 -
.../examples/ml/nn/MLPGroupTrainerExample.java | 141 -
.../examples/ml/nn/MLPLocalTrainerExample.java | 161 -
.../ignite/examples/ml/nn/package-info.java | 22 -
.../apache/ignite/examples/ml/package-info.java | 22 -
...tedLinearRegressionExampleWithQRTrainer.java | 136 -
...edLinearRegressionExampleWithSGDTrainer.java | 137 -
.../ml/regression/linear/package-info.java | 22 -
.../examples/ml/regression/package-info.java | 22 -
.../ignite/examples/ml/trees/MNISTExample.java | 261 --
.../ignite/examples/ml/trees/package-info.java | 22 -
examples/src/main/resources/cityBulkLoad.csv | 3 +
examples/src/main/resources/datasets/README.md | 2 +
.../resources/datasets/cleared_machines.txt | 209 +
examples/src/main/resources/datasets/iris.txt | 150 +
examples/src/main/resources/person.json | 10 +
examples/src/main/resources/personBulkLoad.csv | 4 +
.../examples/ScalarSnowflakeSchemaExample.scala | 4 +-
.../spark/IgniteDataFrameWriteExample.scala | 179 +
.../ignite/examples/BasicExamplesSelfTest.java | 46 +-
.../CacheContinuousQueryExamplesSelfTest.java | 48 +
.../CacheExamplesMultiNodeSelfTest.java | 16 -
.../ignite/examples/CacheExamplesSelfTest.java | 254 +-
.../examples/CheckpointExamplesSelfTest.java | 29 +-
.../examples/ClusterGroupExampleSelfTest.java | 17 +-
.../examples/ContinuationExamplesSelfTest.java | 14 +-
.../ContinuousMapperExamplesSelfTest.java | 12 +-
.../DeploymentExamplesMultiNodeSelfTest.java | 12 +-
.../examples/DeploymentExamplesSelfTest.java | 13 +-
.../ignite/examples/EventsExamplesSelfTest.java | 2 +-
.../ignite/examples/IgfsExamplesSelfTest.java | 38 +-
.../examples/LifecycleExamplesSelfTest.java | 14 +-
.../MemcacheRestExamplesMultiNodeSelfTest.java | 12 +-
.../examples/MemcacheRestExamplesSelfTest.java | 36 +-
.../examples/MessagingExamplesSelfTest.java | 15 +-
.../examples/MonteCarloExamplesSelfTest.java | 14 +-
.../examples/SpringBeanExamplesSelfTest.java | 14 +-
.../ignite/examples/TaskExamplesSelfTest.java | 34 +-
.../IgniteExamplesJ8SelfTestSuite.java | 80 +
.../testsuites/IgniteExamplesMLTestSuite.java | 151 +
.../testsuites/IgniteExamplesSelfTestSuite.java | 7 +-
.../BasicExamplesMultiNodeSelfTest.java | 28 -
.../java8/examples/BasicExamplesSelfTest.java | 89 -
.../CacheExamplesMultiNodeSelfTest.java | 38 -
.../java8/examples/CacheExamplesSelfTest.java | 155 -
.../examples/CheckpointExamplesSelfTest.java | 47 -
.../examples/ClusterGroupExampleSelfTest.java | 40 -
.../ContinuationExamplesMultiNodeSelfTest.java | 28 -
.../examples/ContinuationExamplesSelfTest.java | 37 -
...ntinuousMapperExamplesMultiNodeSelfTest.java | 28 -
.../ContinuousMapperExamplesSelfTest.java | 35 -
.../DeploymentExamplesMultiNodeSelfTest.java | 32 -
.../examples/DeploymentExamplesSelfTest.java | 36 -
.../EventsExamplesMultiNodeSelfTest.java | 28 -
.../java8/examples/EventsExamplesSelfTest.java | 33 -
.../java8/examples/IgfsExamplesSelfTest.java | 55 -
.../examples/LifecycleExamplesSelfTest.java | 37 -
.../MemcacheRestExamplesMultiNodeSelfTest.java | 33 -
.../examples/MemcacheRestExamplesSelfTest.java | 44 -
.../examples/MessagingExamplesSelfTest.java | 55 -
.../MonteCarloExamplesMultiNodeSelfTest.java | 28 -
.../examples/MonteCarloExamplesSelfTest.java | 78 -
.../examples/SpringBeanExamplesSelfTest.java | 37 -
.../examples/TaskExamplesMultiNodeSelfTest.java | 28 -
.../java8/examples/TaskExamplesSelfTest.java | 43 -
.../IgniteExamplesJ8SelfTestSuite.java | 80 -
.../testsuites/IgniteExamplesMLTestSuite.java | 140 -
.../spark/examples/IgniteDataFrameSelfTest.java | 9 +
modules/aop/pom.xml | 13 +-
.../apache/ignite/gridify/BasicAopSelfTest.java | 26 +-
.../testsuites/IgniteAopSelfTestSuite.java | 12 +-
modules/apache-license-gen/pom.xml | 2 +-
modules/aws/pom.xml | 2 +-
modules/benchmarks/pom.xml | 17 +-
.../jmh/notify/JmhWaitStategyBenchmark.java | 7 +-
modules/camel/pom.xml | 2 +-
modules/cassandra/pom.xml | 8 +-
modules/cassandra/serializers/pom.xml | 9 +-
.../cassandra/serializer/package-info.java | 21 +
modules/cassandra/store/pom.xml | 14 +-
.../store/cassandra/CassandraCacheStore.java | 3 +-
.../cassandra/common/PropertyMappingHelper.java | 7 +-
.../cassandra/session/CassandraSessionImpl.java | 9 +-
.../ignite/tests/CassandraSessionImplTest.java | 194 +
.../ignite/tests/load/PersonGenerator.java | 2 +-
.../org/apache/ignite/tests/pojos/Person.java | 12 +-
.../apache/ignite/tests/pojos/SimplePerson.java | 8 +-
.../apache/ignite/tests/utils/TestsHelper.java | 4 +-
modules/clients/pom.xml | 2 +-
.../client/impl/ClientCacheFlagsCodecTest.java | 35 +-
.../jdbc2/JdbcAbstractDmlStatementSelfTest.java | 2 +-
.../internal/jdbc2/JdbcBulkLoadSelfTest.java | 185 +
.../internal/jdbc2/JdbcErrorsSelfTest.java | 12 +-
.../jdbc2/JdbcNoCacheStreamingSelfTest.java | 182 +
.../internal/jdbc2/JdbcStreamingSelfTest.java | 220 +-
.../jdbc2/JdbcStreamingToPublicCacheTest.java | 148 +
.../JettyRestProcessorAbstractSelfTest.java | 1182 ++++--
.../rest/JettyRestProcessorSignedSelfTest.java | 5 +-
.../internal/processors/rest/SimplePerson.java | 4 +-
.../ignite/jdbc/JdbcErrorsAbstractSelfTest.java | 257 +-
.../jdbc/suite/IgniteJdbcDriverTestSuite.java | 34 +-
.../jdbc/thin/JdbcThinAbstractSelfTest.java | 79 +
.../ignite/jdbc/thin/JdbcThinBatchSelfTest.java | 509 ++-
.../thin/JdbcThinBulkLoadAbstractSelfTest.java | 601 +++
...inBulkLoadAtomicPartitionedNearSelfTest.java | 39 +
...bcThinBulkLoadAtomicPartitionedSelfTest.java | 39 +
...dbcThinBulkLoadAtomicReplicatedSelfTest.java | 39 +
...oadTransactionalPartitionedNearSelfTest.java | 39 +
...ulkLoadTransactionalPartitionedSelfTest.java | 39 +
...BulkLoadTransactionalReplicatedSelfTest.java | 39 +
.../thin/JdbcThinComplexDmlDdlSelfTest.java | 2 +-
.../jdbc/thin/JdbcThinConnectionSSLTest.java | 479 +++
.../jdbc/thin/JdbcThinConnectionSelfTest.java | 46 +-
.../JdbcThinDynamicIndexAbstractSelfTest.java | 1 -
.../jdbc/thin/JdbcThinErrorsSelfTest.java | 16 +-
.../jdbc/thin/JdbcThinLocalQueriesSelfTest.java | 73 +
.../jdbc/thin/JdbcThinNoDefaultSchemaTest.java | 2 +-
.../jdbc/thin/JdbcThinStreamingSelfTest.java | 486 +++
.../ignite/jdbc/thin/JdbcThinTcpIoTest.java | 165 +
.../thin/JdbcThinWalModeChangeSelfTest.java | 138 +
.../clients/src/test/resources/bulkload0.csv | 0
.../clients/src/test/resources/bulkload1.csv | 1 +
.../clients/src/test/resources/bulkload2.csv | 2 +
.../src/test/resources/bulkload2_utf.csv | 2 +
modules/cloud/pom.xml | 2 +-
modules/codegen/pom.xml | 11 +-
modules/compatibility/pom.xml | 2 +-
.../DummyPersistenceCompatibilityTest.java | 27 +-
.../FoldersReuseCompatibilityTest.java | 2 +-
.../IgniteUuidCompatibilityTest.java | 189 +
...tingToWalV2SerializerWithCompactionTest.java | 2 +-
.../junits/IgniteCompatibilityNodeRunner.java | 1 +
.../testframework/util/MavenUtils.java | 28 +-
.../IgniteCompatibilityBasicTestSuite.java | 3 +
modules/core/pom.xml | 13 +-
modules/core/src/main/java/META-INF/NOTICE | 2 +-
.../org/apache/ignite/DataRegionMetrics.java | 38 +
.../src/main/java/org/apache/ignite/Ignite.java | 4 +
.../java/org/apache/ignite/IgniteCluster.java | 86 +
.../java/org/apache/ignite/IgniteLogger.java | 92 +-
.../apache/ignite/IgniteSystemProperties.java | 138 +-
.../org/apache/ignite/cache/CacheMetrics.java | 5 +
.../ignite/cache/PartitionLossPolicy.java | 4 +
.../cache/eviction/AbstractEvictionPolicy.java | 4 +-
.../igfs/IgfsPerBlockLruEvictionPolicy.java | 4 +-
.../eviction/sorted/SortedEvictionPolicy.java | 4 +-
.../cache/query/AbstractContinuousQuery.java | 202 +
.../cache/query/BulkLoadContextCursor.java | 97 +
.../ignite/cache/query/ContinuousQuery.java | 172 +-
.../query/ContinuousQueryWithTransformer.java | 192 +
.../cache/store/jdbc/CacheJdbcBlobStore.java | 6 +-
.../org/apache/ignite/cluster/BaselineNode.java | 73 +
.../org/apache/ignite/cluster/ClusterNode.java | 8 +-
.../configuration/AtomicConfiguration.java | 2 +-
.../configuration/CacheConfiguration.java | 29 +-
.../ClientConnectorConfiguration.java | 200 +
.../configuration/DataRegionConfiguration.java | 8 +-
.../configuration/DataStorageConfiguration.java | 35 +-
.../configuration/IgniteConfiguration.java | 37 +
.../configuration/IgniteReflectionFactory.java | 43 +-
.../PersistentStoreConfiguration.java | 7 +-
.../apache/ignite/configuration/WALMode.java | 15 +-
.../org/apache/ignite/internal/ClassSet.java | 108 +
.../apache/ignite/internal/GridComponent.java | 3 +
.../ignite/internal/GridKernalContext.java | 22 +-
.../ignite/internal/GridKernalContextImpl.java | 61 +-
.../apache/ignite/internal/GridLoggerProxy.java | 25 +
.../internal/GridNodeOrderComparator.java | 45 -
.../ignite/internal/GridPluginComponent.java | 5 +
.../org/apache/ignite/internal/GridTopic.java | 3 +
.../apache/ignite/internal/IgniteKernal.java | 205 +-
.../ignite/internal/IgniteVersionUtils.java | 2 +-
.../org/apache/ignite/internal/IgnitionEx.java | 39 +-
.../ignite/internal/LongJVMPauseDetector.java | 167 +
.../ignite/internal/MarshallerContextImpl.java | 41 +-
.../apache/ignite/internal/NodeInvalidator.java | 53 +
.../internal/binary/BinaryClassDescriptor.java | 384 +-
.../ignite/internal/binary/BinaryContext.java | 13 +-
.../ignite/internal/binary/BinaryEnumCache.java | 4 +-
.../internal/binary/BinaryFieldAccessor.java | 61 +-
.../internal/binary/BinaryObjectExImpl.java | 69 +-
.../ignite/internal/binary/BinaryUtils.java | 6 -
.../internal/client/GridClientCacheFlag.java | 48 +
.../internal/client/impl/GridClientImpl.java | 28 +-
.../impl/connection/GridClientConnection.java | 18 +-
.../GridClientConnectionManagerAdapter.java | 16 +-
.../connection/GridClientNioTcpConnection.java | 14 +-
.../GridClientOptimizedMarshaller.java | 4 +-
.../router/impl/GridRouterClientImpl.java | 4 +-
.../client/router/impl/GridTcpRouterImpl.java | 25 +-
.../impl/GridTcpRouterNioListenerAdapter.java | 5 +
.../cluster/ClusterNodeLocalMapImpl.java | 4 +-
.../internal/cluster/DetachedClusterNode.java | 121 +
.../cluster/IgniteClusterAsyncImpl.java | 40 +
.../internal/cluster/IgniteClusterImpl.java | 248 ++
.../internal/cluster/NodeOrderComparator.java | 61 +
.../cluster/NodeOrderLegacyComparator.java | 46 +
.../ignite/internal/commandline/Arguments.java | 117 +
.../internal/commandline/CommandHandler.java | 690 +++-
.../internal/direct/DirectMessageWriter.java | 9 +
.../direct/stream/DirectByteBufferStream.java | 6 +
.../stream/v1/DirectByteBufferStreamImplV1.java | 8 +
.../stream/v2/DirectByteBufferStreamImplV2.java | 11 +
.../internal/events/DiscoveryCustomEvent.java | 34 +
.../ignite/internal/igfs/common/IgfsLogger.java | 12 +-
.../ignite/internal/jdbc/JdbcResultSet.java | 2 +-
.../jdbc/thin/ConnectionProperties.java | 206 +
.../jdbc/thin/ConnectionPropertiesImpl.java | 239 +-
.../internal/jdbc/thin/JdbcThinConnection.java | 117 +
.../jdbc/thin/JdbcThinPreparedStatement.java | 22 +-
.../internal/jdbc/thin/JdbcThinSSLUtil.java | 332 ++
.../internal/jdbc/thin/JdbcThinStatement.java | 219 +-
.../internal/jdbc/thin/JdbcThinTcpIo.java | 139 +-
.../ignite/internal/jdbc2/JdbcConnection.java | 12 +-
.../jdbc2/JdbcQueryMultipleStatementsTask.java | 2 +-
.../ignite/internal/jdbc2/JdbcQueryTask.java | 12 +-
.../ignite/internal/jdbc2/JdbcResultSet.java | 2 +-
.../jdbc2/JdbcStreamedPreparedStatement.java | 2 +-
.../internal/managers/GridManagerAdapter.java | 19 +
.../checkpoint/GridCheckpointManager.java | 4 +-
.../managers/communication/GridIoManager.java | 29 +-
.../communication/GridIoMessageFactory.java | 18 +-
.../managers/deployment/GridDeployment.java | 16 +-
.../deployment/GridDeploymentClassLoader.java | 6 +-
.../deployment/GridDeploymentLocalStore.java | 29 +-
.../GridDeploymentPerVersionStore.java | 6 +-
.../managers/discovery/ClusterMetricsImpl.java | 369 ++
.../managers/discovery/ConsistentIdMapper.java | 96 +-
.../internal/managers/discovery/DiscoCache.java | 138 +-
.../discovery/GridDiscoveryManager.java | 264 +-
.../eventstorage/GridEventStorageManager.java | 4 +-
.../optimized/OptimizedMarshaller.java | 4 +-
.../optimized/OptimizedMarshallerUtils.java | 4 -
.../optimized/OptimizedObjectInputStream.java | 108 +-
.../optimized/OptimizedObjectOutputStream.java | 97 +-
.../mem/unsafe/UnsafeMemoryProvider.java | 12 +-
.../internal/pagemem/PageIdAllocator.java | 5 +-
.../ignite/internal/pagemem/PageMemory.java | 5 +
.../pagemem/impl/PageMemoryNoStoreImpl.java | 7 +-
.../pagemem/store/IgnitePageStoreManager.java | 20 +-
.../pagemem/wal/IgniteWriteAheadLogManager.java | 17 +-
.../internal/pagemem/wal/StorageException.java | 16 +-
.../wal/record/BaselineTopologyRecord.java | 76 +
.../internal/pagemem/wal/record/CacheState.java | 24 +
.../internal/pagemem/wal/record/DataEntry.java | 4 +-
.../internal/pagemem/wal/record/DataRecord.java | 6 +-
.../pagemem/wal/record/ExchangeRecord.java | 86 +
.../pagemem/wal/record/FilteredRecord.java | 3 +
.../pagemem/wal/record/MetastoreDataRecord.java | 62 +
.../pagemem/wal/record/PageSnapshot.java | 12 +-
.../pagemem/wal/record/SnapshotRecord.java | 12 +
.../pagemem/wal/record/TimeStampRecord.java | 8 +
.../internal/pagemem/wal/record/TxRecord.java | 50 +-
.../internal/pagemem/wal/record/WALRecord.java | 37 +-
.../wal/record/WalRecordCacheGroupAware.java | 28 +
.../delta/DataPageInsertFragmentRecord.java | 5 +-
.../wal/record/delta/DataPageInsertRecord.java | 5 +-
.../wal/record/delta/DataPageRemoveRecord.java | 5 +-
.../delta/DataPageSetFreeListPageRecord.java | 5 +-
.../wal/record/delta/DataPageUpdateRecord.java | 5 +-
.../wal/record/delta/PageDeltaRecord.java | 9 +-
.../record/delta/PartitionDestroyRecord.java | 9 +-
.../record/delta/PartitionMetaStateRecord.java | 9 +-
.../processors/GridProcessorAdapter.java | 5 +
.../affinity/GridAffinityAssignmentCache.java | 108 +-
.../affinity/GridAffinityProcessor.java | 4 +-
.../bulkload/BulkLoadAckClientParameters.java | 92 +
.../bulkload/BulkLoadCacheWriter.java | 31 +
.../processors/bulkload/BulkLoadCsvFormat.java | 159 +
.../processors/bulkload/BulkLoadCsvParser.java | 65 +
.../processors/bulkload/BulkLoadFormat.java | 33 +
.../processors/bulkload/BulkLoadParser.java | 61 +
.../processors/bulkload/BulkLoadProcessor.java | 104 +
.../bulkload/BulkLoadStreamerWriter.java | 65 +
.../bulkload/pipeline/CharsetDecoderBlock.java | 132 +
.../pipeline/CsvLineProcessorBlock.java | 70 +
.../bulkload/pipeline/LineSplitterBlock.java | 72 +
.../bulkload/pipeline/PipelineBlock.java | 66 +
.../bulkload/pipeline/StrListAppenderBlock.java | 52 +
.../cache/CacheAffinitySharedManager.java | 188 +-
.../cache/CacheClusterMetricsMXBeanImpl.java | 5 +
.../cache/CacheConfigurationOverride.java | 165 +
.../processors/cache/CacheGroupContext.java | 97 +-
.../processors/cache/CacheGroupData.java | 36 +-
.../processors/cache/CacheGroupDescriptor.java | 75 +-
.../cache/CacheJoinNodeDiscoveryData.java | 2 +-
.../cache/CacheLocalMetricsMXBeanImpl.java | 5 +
.../processors/cache/CacheMetricsImpl.java | 332 +-
.../processors/cache/CacheMetricsSnapshot.java | 35 +-
.../cache/CacheStoreBalancingWrapper.java | 4 +-
.../cache/CacheWeakQueryIteratorsHolder.java | 4 +-
.../processors/cache/ClusterCachesInfo.java | 77 +-
.../cache/DynamicCacheDescriptor.java | 19 +-
.../processors/cache/ExchangeActions.java | 33 +-
.../cache/ExchangeDiscoveryEvents.java | 29 +-
.../cache/GatewayProtectedCacheProxy.java | 14 +
.../processors/cache/GridCacheAdapter.java | 26 +-
.../cache/GridCacheConcurrentMapImpl.java | 14 +-
.../processors/cache/GridCacheContext.java | 72 +-
.../cache/GridCacheDeploymentManager.java | 6 +-
.../processors/cache/GridCacheEntryEx.java | 24 +
.../processors/cache/GridCacheEventManager.java | 3 +-
.../processors/cache/GridCacheIoManager.java | 12 +-
.../cache/GridCacheLocalConcurrentMap.java | 4 +-
.../processors/cache/GridCacheMapEntry.java | 1034 +++--
.../processors/cache/GridCacheMvccManager.java | 13 +-
.../GridCachePartitionExchangeManager.java | 93 +-
.../processors/cache/GridCachePreloader.java | 16 +-
.../cache/GridCachePreloaderAdapter.java | 12 +-
.../processors/cache/GridCacheProcessor.java | 412 +-
.../cache/GridCacheSharedContext.java | 58 +-
.../cache/GridCacheSharedManager.java | 5 +
.../cache/GridCacheSharedManagerAdapter.java | 5 +
.../processors/cache/GridCacheTtlManager.java | 8 +-
.../cache/GridCacheUpdateTxResult.java | 21 +-
.../processors/cache/GridCacheUtils.java | 24 +-
.../cache/GridDeferredAckMessageSender.java | 12 +-
.../cache/IgniteCacheOffheapManager.java | 14 +-
.../cache/IgniteCacheOffheapManagerImpl.java | 126 +-
.../processors/cache/IgniteCacheProxy.java | 41 +-
.../processors/cache/IgniteCacheProxyImpl.java | 172 +-
.../cache/IgniteRebalanceIterator.java | 23 +-
.../cache/LocalJoinCachesContext.java | 127 +
.../processors/cache/StateChangeRequest.java | 42 +
.../processors/cache/StoredCacheData.java | 9 +-
.../cache/WalStateAbstractMessage.java | 135 +
.../processors/cache/WalStateAckMessage.java | 221 +
.../cache/WalStateDistributedProcess.java | 172 +
.../processors/cache/WalStateFinishMessage.java | 73 +
.../processors/cache/WalStateManager.java | 914 +++++
.../cache/WalStateNodeLeaveExchangeTask.java | 57 +
.../cache/WalStateProposeMessage.java | 104 +
.../processors/cache/WalStateResult.java | 94 +
.../cache/binary/BinaryMetadataFileStore.java | 4 +-
.../cache/binary/BinaryMetadataTransport.java | 14 +-
.../binary/CacheObjectBinaryProcessorImpl.java | 123 +-
.../CacheDataStructuresManager.java | 8 +-
.../distributed/GridCacheTxFinishSync.java | 6 +-
.../distributed/GridDistributedCacheEntry.java | 84 +-
.../GridDistributedTxRemoteAdapter.java | 19 +-
.../dht/GridClientPartitionTopology.java | 34 +-
.../dht/GridDhtAssignmentFetchFuture.java | 4 +-
.../distributed/dht/GridDhtCacheAdapter.java | 8 +-
.../distributed/dht/GridDhtCacheEntry.java | 295 +-
.../distributed/dht/GridDhtLocalPartition.java | 488 ++-
.../distributed/dht/GridDhtLockFuture.java | 4 +-
.../dht/GridDhtPartitionTopology.java | 10 +-
.../dht/GridDhtPartitionTopologyImpl.java | 335 +-
.../dht/GridDhtPartitionsEvictor.java | 140 +
.../dht/GridDhtPartitionsReservation.java | 6 +-
.../dht/GridDhtTopologyFutureAdapter.java | 2 +-
.../distributed/dht/GridDhtTxLocalAdapter.java | 14 +-
.../distributed/dht/GridDhtTxPrepareFuture.java | 5 +
.../dht/GridPartitionedGetFuture.java | 13 +-
.../dht/GridPartitionedSingleGetFuture.java | 18 +-
.../dht/atomic/GridDhtAtomicCache.java | 13 +-
.../atomic/GridNearAtomicFullUpdateRequest.java | 1 +
.../colocated/GridDhtColocatedLockFuture.java | 4 +-
.../colocated/GridDhtDetachedCacheEntry.java | 19 +-
.../CachePartitionFullCountersMap.java | 22 +
.../CachePartitionPartialCountersMap.java | 61 +-
.../GridDhtPartitionDemandLegacyMessage.java | 435 ++
.../GridDhtPartitionDemandMessage.java | 201 +-
.../dht/preloader/GridDhtPartitionDemander.java | 445 +-
.../preloader/GridDhtPartitionExchangeId.java | 3 +
.../dht/preloader/GridDhtPartitionSupplier.java | 548 ++-
.../GridDhtPartitionSupplyMessage.java | 22 +-
.../GridDhtPartitionsExchangeFuture.java | 446 +-
.../dht/preloader/GridDhtPreloader.java | 198 +-
.../IgniteDhtDemandedPartitionsMap.java | 218 +
.../IgniteDhtPartitionCountersMap.java | 6 +
.../IgniteDhtPartitionHistorySuppliersMap.java | 6 +
.../IgniteDhtPartitionsToReloadMap.java | 13 +
.../dht/preloader/IgniteHistoricalIterator.java | 39 +
.../preloader/IgniteRebalanceIteratorImpl.java | 201 +
...StopCachesOnClientReconnectExchangeTask.java | 59 +
.../distributed/near/GridNearCacheEntry.java | 220 +-
.../distributed/near/GridNearGetFuture.java | 13 +-
.../distributed/near/GridNearLockFuture.java | 4 +-
.../near/GridNearOptimisticTxPrepareFuture.java | 61 +-
...ridNearOptimisticTxPrepareFutureAdapter.java | 2 +-
.../near/GridNearTxPrepareFutureAdapter.java | 2 +-
.../distributed/near/IgniteTxMappingsImpl.java | 4 +-
.../cache/local/GridLocalCacheEntry.java | 50 +-
.../local/atomic/GridLocalAtomicCache.java | 9 +-
.../cache/persistence/AllocatedPageTracker.java | 30 +
.../cache/persistence/CacheDataRow.java | 2 +-
.../cache/persistence/CheckpointFuture.java | 35 +
.../CheckpointWriteProgressSupplier.java | 45 +
.../persistence/DataRegionMetricsImpl.java | 100 +-
.../DataRegionMetricsMXBeanImpl.java | 25 +
.../persistence/DataRegionMetricsSnapshot.java | 45 +
.../cache/persistence/DataStructure.java | 17 +-
.../GridCacheDatabaseSharedManager.java | 1660 ++++++--
.../persistence/GridCacheOffheapManager.java | 617 ++-
.../IgniteCacheDatabaseSharedManager.java | 136 +-
.../cache/persistence/IndexStorage.java | 51 +
.../cache/persistence/IndexStorageImpl.java | 420 ++
.../processors/cache/persistence/MetaStore.java | 51 -
.../cache/persistence/MetadataStorage.java | 420 --
.../cache/persistence/RecoveryDebug.java | 170 +
.../processors/cache/persistence/Storable.java | 38 +
.../cache/persistence/file/FileDownloader.java | 189 +
.../cache/persistence/file/FileIO.java | 7 +-
.../cache/persistence/file/FilePageStore.java | 139 +-
.../persistence/file/FilePageStoreFactory.java | 3 +-
.../persistence/file/FilePageStoreManager.java | 275 +-
.../cache/persistence/file/FilePageStoreV2.java | 11 +-
.../cache/persistence/file/FileUploader.java | 100 +
.../file/FileVersionCheckingFactory.java | 51 +-
.../file/PersistentStorageIOException.java | 47 +
.../cache/persistence/file/UnzipFileIO.java | 141 +
.../filename/PdsConsistentIdProcessor.java | 3 +
.../persistence/freelist/AbstractFreeList.java | 603 +++
.../persistence/freelist/CacheFreeListImpl.java | 59 +
.../cache/persistence/freelist/FreeList.java | 8 +-
.../persistence/freelist/FreeListImpl.java | 619 ---
.../cache/persistence/freelist/PagesList.java | 55 +-
.../persistence/metastorage/MetaStorage.java | 547 +++
.../metastorage/MetastorageDataRow.java | 91 +
.../MetastorageLifecycleListener.java | 47 +
.../metastorage/MetastorageRowStore.java | 96 +
.../metastorage/MetastorageSearchRow.java | 38 +
.../metastorage/MetastorageTree.java | 266 ++
.../metastorage/MetsatorageSearchRowImpl.java | 55 +
.../metastorage/ReadOnlyMetastorage.java | 40 +
.../metastorage/ReadWriteMetastorage.java | 32 +
.../pagemem/CheckpointMetricsTracker.java | 2 +-
.../pagemem/DelayedDirtyPageWrite.java | 105 +
.../pagemem/DelayedPageReplacementTracker.java | 198 +
.../persistence/pagemem/EvictCandidate.java | 77 -
.../persistence/pagemem/FullPageIdTable.java | 358 +-
.../pagemem/IntervalBasedMeasurement.java | 304 ++
.../persistence/pagemem/LoadedPagesMap.java | 137 +
.../cache/persistence/pagemem/PageMemoryEx.java | 36 +-
.../persistence/pagemem/PageMemoryImpl.java | 736 ++--
.../pagemem/PagesWriteSpeedBasedThrottle.java | 529 +++
.../persistence/pagemem/PagesWriteThrottle.java | 83 +-
.../pagemem/PagesWriteThrottlePolicy.java | 39 +
.../persistence/pagemem/ReplaceCandidate.java | 78 +
.../persistence/pagemem/ReplacedPageWriter.java | 35 +
.../pagemem/RobinHoodBackwardShiftHashMap.java | 657 +++
.../partstate/PagesAllocationRange.java | 7 +-
.../partstate/PartitionAllocationMap.java | 36 +-
.../snapshot/IgniteCacheSnapshotManager.java | 23 +-
.../snapshot/SnapshotDiscoveryMessage.java | 7 +-
.../cache/persistence/tree/BPlusTree.java | 15 +
.../persistence/tree/io/AbstractDataPageIO.java | 1256 ++++++
.../cache/persistence/tree/io/DataPageIO.java | 1214 +-----
.../cache/persistence/tree/io/PageIO.java | 26 +-
.../tree/io/PagePartitionCountersIO.java | 34 +-
.../tree/io/PagePartitionMetaIO.java | 17 +-
.../persistence/tree/io/SimpleDataPageIO.java | 127 +
.../cache/persistence/tree/reuse/ReuseBag.java | 5 +
.../persistence/tree/util/PageHandler.java | 2 +-
.../wal/AbstractWalRecordsIterator.java | 97 +-
.../wal/FileWriteAheadLogManager.java | 719 ++--
.../wal/FsyncModeFileWriteAheadLogManager.java | 3226 +++++++++++++++
.../persistence/wal/SegmentArchivedMonitor.java | 64 +
.../wal/SegmentReservationStorage.java | 61 +
.../SingleSegmentLogicalRecordsIterator.java | 12 +-
.../wal/reader/IgniteWalIteratorFactory.java | 42 +-
.../wal/reader/StandaloneGridKernalContext.java | 18 +-
.../reader/StandaloneWalRecordsIterator.java | 78 +-
.../BaselineTopologyRecordSerializer.java | 168 +
.../wal/serializer/RecordDataV1Serializer.java | 80 +-
.../wal/serializer/RecordDataV2Serializer.java | 224 +-
.../wal/serializer/RecordV1Serializer.java | 6 +-
.../wal/serializer/RecordV2Serializer.java | 11 +-
.../wal/serializer/TxRecordSerializer.java | 152 +-
.../query/GridCacheDistributedQueryManager.java | 6 +-
.../cache/query/GridCacheQueryManager.java | 331 +-
.../query/GridCacheQueryMetricsAdapter.java | 10 +-
.../cache/query/SqlFieldsQueryEx.java | 45 +
.../continuous/CacheContinuousQueryEntry.java | 9 +-
.../CacheContinuousQueryEventBuffer.java | 5 +-
.../continuous/CacheContinuousQueryHandler.java | 153 +-
.../CacheContinuousQueryHandlerV2.java | 9 +-
.../CacheContinuousQueryHandlerV3.java | 185 +
.../continuous/CacheContinuousQueryManager.java | 56 +-
.../processors/cache/store/CacheLocalStore.java | 4 +
.../store/GridCacheStoreManagerAdapter.java | 6 +
.../cache/store/GridCacheWriteBehindStore.java | 11 +-
.../cache/transactions/IgniteTxAdapter.java | 73 +-
.../transactions/IgniteTxLocalAdapter.java | 40 +-
.../cache/transactions/IgniteTxManager.java | 21 +-
.../CollectConflictPartitionKeysTask.java | 235 ++
.../cache/verify/PartitionEntryHashRecord.java | 237 ++
.../cache/verify/PartitionHashRecord.java | 130 +
.../processors/cache/verify/PartitionKey.java | 103 +
.../RetrieveConflictPartitionValuesTask.java | 204 +
.../verify/VerifyBackupPartitionsTask.java | 359 ++
.../processors/cluster/BaselineTopology.java | 484 +++
.../cluster/BaselineTopologyHistory.java | 127 +
.../cluster/BaselineTopologyHistoryItem.java | 79 +
.../processors/cluster/BranchingPointType.java | 48 +
.../cluster/ChangeGlobalStateFinishMessage.java | 19 +-
.../cluster/ChangeGlobalStateMessage.java | 38 +-
.../cluster/DiscoveryDataClusterState.java | 127 +-
.../cluster/GridClusterStateProcessor.java | 781 +++-
.../cluster/IGridClusterStateProcessor.java | 122 +
.../continuous/GridContinuousBatchAdapter.java | 5 +-
.../continuous/GridContinuousProcessor.java | 16 +-
.../datastreamer/DataStreamerImpl.java | 57 +-
.../AtomicDataStructureProxy.java | 4 +-
.../datastructures/DataStructuresProcessor.java | 109 +-
.../processors/hadoop/HadoopClassLoader.java | 5 +-
.../processors/igfs/IgfsClientSession.java | 4 +-
.../processors/igfs/IgfsDataManager.java | 10 +-
.../internal/processors/igfs/IgfsImpl.java | 4 +-
.../processors/igfs/IgfsLocalMetrics.java | 30 +-
.../internal/processors/igfs/IgfsProcessor.java | 4 +-
.../processors/job/GridJobProcessor.java | 18 +-
.../jobmetrics/GridJobMetricsProcessor.java | 4 +-
.../GridMarshallerMappingProcessor.java | 6 +-
.../odbc/ClientListenerBufferedParser.java | 6 +-
.../odbc/ClientListenerNioListener.java | 113 +-
.../odbc/ClientListenerNioServerBuffer.java | 12 +-
.../odbc/ClientListenerProcessor.java | 88 +-
.../odbc/jdbc/JdbcBatchExecuteRequest.java | 40 +-
.../odbc/jdbc/JdbcBulkLoadAckResult.java | 111 +
.../odbc/jdbc/JdbcBulkLoadBatchRequest.java | 183 +
.../odbc/jdbc/JdbcBulkLoadProcessor.java | 144 +
.../odbc/jdbc/JdbcConnectionContext.java | 13 +-
.../processors/odbc/jdbc/JdbcRequest.java | 7 +
.../odbc/jdbc/JdbcRequestHandler.java | 272 +-
.../processors/odbc/jdbc/JdbcResult.java | 8 +
.../odbc/odbc/OdbcRequestHandler.java | 90 +-
.../offheap/GridOffHeapProcessor.java | 6 +-
.../platform/PlatformContextImpl.java | 20 +-
.../platform/PlatformProcessorImpl.java | 83 +-
.../platform/cache/PlatformCache.java | 1 +
.../cache/ClientCacheSqlFieldsQueryRequest.java | 8 +-
.../platform/cluster/PlatformClusterGroup.java | 15 +-
.../utils/PlatformConfigurationUtils.java | 79 +-
.../platform/utils/PlatformUtils.java | 51 +-
.../processors/query/GridQueryIndexing.java | 44 +-
.../processors/query/GridQueryProcessor.java | 178 +-
.../processors/query/IgniteSQLException.java | 9 +
.../internal/processors/query/QueryUtils.java | 6 +-
.../processors/query/SqlClientContext.java | 223 +
.../processors/resource/GridResourceIoc.java | 4 +-
.../processors/rest/GridRestProcessor.java | 6 +-
.../handlers/cache/GridCacheCommandHandler.java | 148 +-
.../cluster/GridChangeStateCommandHandler.java | 4 +-
.../top/GridTopologyCommandHandler.java | 2 +-
.../rest/protocols/GridRestProtocolAdapter.java | 5 +-
.../rest/protocols/tcp/GridTcpRestParser.java | 15 +-
.../rest/protocols/tcp/GridTcpRestProtocol.java | 31 +-
.../rest/request/GridRestCacheRequest.java | 40 +-
.../service/GridServiceProcessor.java | 13 +-
.../processors/service/GridServiceProxy.java | 6 +-
.../session/GridTaskSessionProcessor.java | 4 +-
.../GridInternalSubscriptionProcessor.java | 57 +
.../processors/task/GridTaskProcessor.java | 4 +-
.../processors/task/GridTaskWorker.java | 4 +-
.../apache/ignite/internal/sql/SqlKeyword.java | 53 +-
.../apache/ignite/internal/sql/SqlParser.java | 71 +-
.../ignite/internal/sql/SqlParserUtils.java | 22 +
.../sql/command/SqlAlterTableCommand.java | 114 +
.../sql/command/SqlBulkLoadCommand.java | 279 ++
.../sql/command/SqlSetStreamingCommand.java | 191 +
.../ignite/internal/util/Base64Encoder.java | 33 -
.../ignite/internal/util/Base64EncoderImpl.java | 45 -
.../internal/util/GridAnnotationsCache.java | 6 +-
.../internal/util/GridClassLoaderCache.java | 4 +-
.../internal/util/GridConcurrentFactory.java | 4 +-
.../internal/util/GridConcurrentHashSet.java | 10 +-
.../ignite/internal/util/GridIntList.java | 2 +-
.../ignite/internal/util/GridKeyLock.java | 4 +-
.../ignite/internal/util/GridLogThrottle.java | 4 +-
.../internal/util/GridSynchronizedMap.java | 4 +-
.../apache/ignite/internal/util/GridUnsafe.java | 36 +-
.../ignite/internal/util/IgniteUtils.java | 213 +-
.../internal/util/LegacyBase64Encoder.java | 42 -
.../apache/ignite/internal/util/MpscQueue.java | 240 --
.../ignite/internal/util/StripedExecutor.java | 304 +-
.../util/lang/GridFilteredClosableIterator.java | 37 +
.../util/lang/GridFilteredIterator.java | 2 +-
.../ignite/internal/util/lang/GridFunc.java | 50 +-
.../ConcurrentDequeFactoryCallable.java | 40 -
.../gridfunc/ConcurrentMapFactoryCallable.java | 4 +-
.../ignite/internal/util/nio/GridNioServer.java | 48 +-
.../util/nio/GridNioServerListener.java | 8 +
.../util/nio/GridNioServerListenerAdapter.java | 5 +
.../util/nio/GridSelectorNioSessionImpl.java | 5 +-
.../util/offheap/unsafe/GridUnsafeMap.java | 8 +-
.../unsafe/GridUnsafePartitionedMap.java | 4 +-
.../internal/visor/VisorMultiNodeTask.java | 6 +
.../visor/baseline/VisorBaselineNode.java | 89 +
.../visor/baseline/VisorBaselineOperation.java | 49 +
.../visor/baseline/VisorBaselineTask.java | 224 ++
.../visor/baseline/VisorBaselineTaskArg.java | 99 +
.../visor/baseline/VisorBaselineTaskResult.java | 144 +
.../visor/cache/VisorMemoryMetrics.java | 74 +
.../visor/compute/VisorGatewayTask.java | 16 +-
.../node/VisorClientConnectorConfiguration.java | 117 +-
.../node/VisorDataStorageConfiguration.java | 47 +-
.../internal/visor/query/VisorQueryTask.java | 14 +-
.../internal/visor/query/VisorQueryUtils.java | 9 +
.../internal/visor/util/VisorTaskUtils.java | 8 +-
.../apache/ignite/logger/java/JavaLogger.java | 2 +-
.../ignite/marshaller/MarshallerContext.java | 14 +
.../ignite/marshaller/MarshallerUtils.java | 6 +
.../ignite/marshaller/jdk/JdkMarshaller.java | 20 +-
.../jdk/JdkMarshallerObjectInputStream.java | 9 +-
.../org/apache/ignite/mxbean/IgniteMXBean.java | 25 +
.../extensions/communication/MessageWriter.java | 10 +
.../org/apache/ignite/spi/IgniteSpiAdapter.java | 6 +
.../org/apache/ignite/spi/IgniteSpiContext.java | 7 +
.../jobstealing/JobStealingCollisionSpi.java | 10 +-
.../tcp/TcpCommunicationMetricsListener.java | 312 +-
.../communication/tcp/TcpCommunicationSpi.java | 18 +-
.../deployment/local/LocalDeploymentSpi.java | 4 +-
.../ignite/spi/discovery/tcp/ClientImpl.java | 27 +-
.../ignite/spi/discovery/tcp/ServerImpl.java | 29 +-
.../spi/discovery/tcp/TcpDiscoveryImpl.java | 7 +-
.../spi/discovery/tcp/TcpDiscoverySpi.java | 64 +-
.../tcp/internal/DiscoveryDataPacket.java | 14 +-
.../tcp/internal/TcpDiscoveryNode.java | 7 +-
.../adaptive/AdaptiveLoadBalancingSpi.java | 4 +-
.../roundrobin/RoundRobinLoadBalancingSpi.java | 4 +-
.../WeightedRandomLoadBalancingSpi.java | 4 +-
.../org/apache/ignite/stream/StreamAdapter.java | 2 +-
.../apache/ignite/stream/StreamTransformer.java | 60 +-
.../ignite/stream/socket/SocketStreamer.java | 8 +-
.../apache/ignite/util/deque/FastSizeDeque.java | 372 ++
.../java/org/jsr166/ConcurrentHashMap8.java | 3808 ------------------
.../java/org/jsr166/ConcurrentLinkedDeque8.java | 3 +-
.../org/jsr166/ConcurrentLinkedHashMap.java | 5 +-
.../src/main/java/org/jsr166/LongAdder8.java | 211 -
.../src/main/java/org/jsr166/Striped64_8.java | 350 --
.../java/org/jsr166/ThreadLocalRandom8.java | 207 -
.../resources/META-INF/classnames.properties | 107 +-
.../core/src/main/resources/ignite.properties | 2 +-
.../test/config/class_list_exploit_excluded.txt | 18 +
.../test/config/class_list_exploit_included.txt | 19 +
modules/core/src/test/config/log4j2-test.xml | 6 +-
.../src/test/config/log4j2-verbose-test.xml | 4 +-
...CacheJdbcBlobStoreMultithreadedSelfTest.java | 6 +-
.../apache/ignite/internal/ClassSetTest.java | 71 +
.../ignite/internal/GridVersionSelfTest.java | 2 -
.../IgniteClientReconnectApiExceptionTest.java | 26 +-
.../IgniteClientReconnectCacheTest.java | 6 +-
.../IgniteReflectionFactorySelfTest.java | 235 ++
...UpdateNotifierPerClusterSettingSelfTest.java | 2 -
.../MarshallerContextLockingSelfTest.java | 2 +-
.../internal/TestDelayingCommunicationSpi.java | 4 +-
.../binary/BinaryMarshallerSelfTest.java | 3 +-
.../binary/BinaryObjectExceptionSelfTest.java | 41 +
.../internal/binary/BinaryTreeSelfTest.java | 273 +-
...GridBinaryMarshallerCtxDisabledSelfTest.java | 12 +
.../commandline/CommandHandlerParsingTest.java | 118 +
.../GridDeploymentMessageCountSelfTest.java | 4 +-
.../OptimizedObjectStreamSelfTest.java | 96 +-
.../pagemem/impl/PageMemoryNoLoadSelfTest.java | 2 +-
.../cache/CacheMetricsEnableRuntimeTest.java | 4 +-
.../cache/CacheMetricsEntitiesCountTest.java | 172 +
.../cache/CacheNearReaderUpdateTest.java | 4 +-
.../CacheSerializableTransactionsTest.java | 4 +-
.../DataStorageConfigurationValidationTest.java | 46 +
.../cache/GridCacheAbstractFullApiSelfTest.java | 11 +-
.../GridCacheAbstractRemoveFailureTest.java | 80 +-
.../cache/GridCacheAbstractSelfTest.java | 3 +-
.../cache/GridCacheEntryMemorySizeSelfTest.java | 11 +-
.../GridCacheInterceptorAbstractSelfTest.java | 12 +-
.../GridCacheOrderedPreloadingSelfTest.java | 8 +-
...ridCacheReplicatedSynchronousCommitTest.java | 4 +-
...ridCacheStoreManagerDeserializationTest.java | 4 +-
.../processors/cache/GridCacheTestEntryEx.java | 19 +-
.../cache/IgniteCacheAbstractTest.java | 6 +-
.../IgniteCacheReadThroughStoreCallTest.java | 4 +-
...IgniteClientCacheInitializationFailTest.java | 20 +-
.../IgniteClusterActivateDeactivateTest.java | 251 +-
...erActivateDeactivateTestWithPersistence.java | 79 +-
.../cache/IgniteDynamicCacheStartSelfTest.java | 8 +
.../IgniteMarshallerCacheFSRestoreTest.java | 2 +-
.../IgniteOnePhaseCommitNearReadersTest.java | 2 +-
.../processors/cache/MapCacheStoreStrategy.java | 4 +-
.../cache/WalModeChangeAbstractSelfTest.java | 282 ++
.../cache/WalModeChangeAdvancedSelfTest.java | 533 +++
.../WalModeChangeCommonAbstractSelfTest.java | 328 ++
...hangeCoordinatorNotAffinityNodeSelfTest.java | 30 +
.../processors/cache/WalModeChangeSelfTest.java | 30 +
...naryObjectsAbstractDataStreamerSelfTest.java | 14 +-
...aryObjectsAbstractMultiThreadedSelfTest.java | 18 +-
.../GridCacheBinaryStoreAbstractSelfTest.java | 4 +-
...eAbstractDataStructuresFailoverSelfTest.java | 77 +
.../IgniteClientDataStructuresAbstractTest.java | 40 +-
.../distributed/Cache64kPartitionsTest.java | 2 +-
.../distributed/CacheBaselineTopologyTest.java | 950 +++++
.../distributed/CacheExchangeMergeTest.java | 10 +-
.../CacheLateAffinityAssignmentTest.java | 5 +-
.../CachePutAllFailoverAbstractTest.java | 35 +-
.../distributed/GridCacheEventAbstractTest.java | 82 +-
...chePartitionedReloadAllAbstractSelfTest.java | 4 +-
.../distributed/IgniteCache150ClientsTest.java | 4 +-
.../IgniteCachePartitionLossPolicySelfTest.java | 31 +-
.../IgniteOptimisticTxSuspendResumeTest.java | 52 +
...achePartitionPartialCountersMapSelfTest.java | 57 +
.../dht/GridCacheGlobalLoadTest.java | 6 +-
...ridCachePartitionedUnloadEventsSelfTest.java | 25 +-
.../dht/IgniteCrossCacheTxSelfTest.java | 8 +-
...cingDelayedPartitionMapExchangeSelfTest.java | 4 +-
...idCacheRebalancingPartitionCountersTest.java | 180 +
.../GridCacheRebalancingSyncSelfTest.java | 3 +-
...idCacheRebalancingWithAsyncClearingTest.java | 240 ++
...ridCacheReplicatedEventDisabledSelfTest.java | 75 +
.../GridCacheReplicatedPreloadSelfTest.java | 7 +
.../eviction/paged/PageEvictionMetricTest.java | 69 +
.../PageEvictionMultinodeMixedRegionsTest.java | 9 +-
.../SortedEvictionPolicyPerformanceTest.java | 8 +-
.../IgniteCacheLoadAllAbstractTest.java | 6 +-
.../IgniteCacheLoaderWriterAbstractTest.java | 4 +-
...eBaselineAffinityTopologyActivationTest.java | 993 +++++
.../IgniteDataStorageMetricsSelfTest.java | 8 +-
...tePdsBinaryMetadataOnClusterRestartTest.java | 272 +-
.../IgnitePdsBinarySortObjectFieldsTest.java | 6 +-
...gnitePdsCacheAssignmentNodeRestartsTest.java | 4 +-
.../IgnitePdsCacheRebalancingAbstractTest.java | 89 +-
.../IgnitePdsClientNearCachePutGetTest.java | 33 +-
.../IgnitePdsContinuousRestartTest.java | 13 +-
.../IgnitePdsContinuousRestartTest2.java | 281 ++
.../IgnitePdsCorruptedStoreTest.java | 154 +
.../persistence/IgnitePdsDynamicCacheTest.java | 15 +-
.../IgnitePdsExchangeDuringCheckpointTest.java | 6 +-
...MarshallerMappingRestoreOnNodeStartTest.java | 5 +-
.../IgnitePdsMultiNodePutGetRestartTest.java | 6 +-
.../persistence/IgnitePdsPageSizesTest.java | 7 +-
...gnitePdsRecoveryAfterFileCorruptionTest.java | 13 +-
.../IgnitePdsRemoveDuringRebalancingTest.java | 4 +-
...gnitePdsSingleNodePutGetPersistenceTest.java | 22 +-
.../IgnitePersistentStoreCacheGroupsTest.java | 6 +-
...IgnitePersistentStoreDataStructuresTest.java | 6 +-
...teAllBaselineNodesOnlineFullApiSelfTest.java | 35 +
.../IgniteBaselineAbstractFullApiSelfTest.java | 52 +
...gingBaselineDownCachePutAllFailoverTest.java | 127 +
...gingBaselineDownCacheRemoveFailoverTest.java | 157 +
...angingBaselineUpCachePutAllFailoverTest.java | 122 +
...angingBaselineUpCacheRemoveFailoverTest.java | 154 +
...gniteOfflineBaselineNodeFullApiSelfTest.java | 39 +
...eOnlineNodeOutOfBaselineFullApiSelfTest.java | 37 +
...teStableBaselineCachePutAllFailoverTest.java | 105 +
...teStableBaselineCacheRemoveFailoverTest.java | 131 +
.../db/IgnitePdsCacheRestoreTest.java | 6 +-
.../db/IgnitePdsDataRegionMetricsTest.java | 232 ++
.../db/IgnitePdsMultiNodePutGetRestartTest.java | 6 +-
...PdsPageEvictionDuringPartitionClearTest.java | 6 +-
.../db/IgnitePdsPageEvictionTest.java | 7 +-
...tePdsRebalancingOnNotStableTopologyTest.java | 6 +-
.../db/IgnitePdsTransactionsHangTest.java | 9 +-
.../db/IgnitePdsWholeClusterRestartTest.java | 16 +-
...gniteCheckpointDirtyPagesForLowLoadTest.java | 205 +
.../checkpoint/IgniteMassLoadSandboxTest.java | 517 +++
.../db/checkpoint/ProgressWatchdog.java | 495 +++
...faultPageSizeBackwardsCompatibilityTest.java | 14 +-
...nitePdsCacheDestroyDuringCheckpointTest.java | 154 +
.../db/file/IgnitePdsCacheIntegrationTest.java | 7 +-
.../db/file/IgnitePdsCheckpointSimpleTest.java | 100 +
...ckpointSimulationWithRealCpDisabledTest.java | 99 +-
.../file/IgnitePdsDiskErrorsRecoveringTest.java | 362 ++
.../db/file/IgnitePdsEvictionTest.java | 331 --
.../file/IgnitePdsNoActualWalHistoryTest.java | 7 +-
.../db/file/IgnitePdsPageReplacementTest.java | 329 ++
.../file/IgnitePdsThreadInterruptionTest.java | 157 +-
.../IgniteUidAsConsistentIdMigrationTest.java | 11 +-
.../db/wal/IgniteWalFlushDefaultSelfTest.java | 2 +-
.../db/wal/IgniteWalFlushFailoverTest.java | 40 +-
...lFlushMultiNodeFailoverAbstractSelfTest.java | 38 +-
.../wal/IgniteWalHistoryReservationsTest.java | 70 +-
.../db/wal/IgniteWalRecoveryPPCTest.java | 4 +-
.../IgniteWalRecoverySeveralRestartsTest.java | 6 +-
.../db/wal/IgniteWalRecoveryTest.java | 288 +-
.../db/wal/IgniteWalSerializerVersionTest.java | 42 +-
.../persistence/db/wal/WalCompactionTest.java | 12 +-
.../db/wal/WalRecoveryTxLogicalRecordsTest.java | 121 +-
.../db/wal/reader/IgniteWalReaderTest.java | 50 +-
.../db/wal/reader/MockWalIteratorFactory.java | 2 +-
.../persistence/file/FileDownloaderTest.java | 90 +
.../metastorage/IgniteMetaStorageBasicTest.java | 147 +
.../pagemem/BPlusTreePageMemoryImplTest.java | 20 +-
.../BPlusTreeReuseListPageMemoryImplTest.java | 20 +-
.../pagemem/FillFactorMetricTest.java | 214 +
.../pagemem/FullPageIdTableTest.java | 195 +-
...gnitePageMemReplaceDelayedWriteUnitTest.java | 306 ++
.../pagemem/IgniteThrottlingUnitTest.java | 273 ++
.../pagemem/IndexStoragePageMemoryImplTest.java | 98 +
.../MetadataStoragePageMemoryImplTest.java | 104 -
.../pagemem/NoOpPageStoreManager.java | 20 +
.../persistence/pagemem/NoOpWALManager.java | 17 +-
.../pagemem/PageMemoryImplNoLoadTest.java | 14 +-
.../persistence/pagemem/PageMemoryImplTest.java | 219 +-
.../pagemem/PagesWriteThrottleSandboxTest.java | 8 +-
.../pagemem/PagesWriteThrottleSmokeTest.java | 11 +-
.../RobinHoodBackwardShiftHashMapTest.java | 442 ++
.../AbstractNodeJoinTemplate.java | 6 +-
.../IgniteChangeGlobalStateAbstractTest.java | 5 +-
.../IgniteChangeGlobalStateFailOverTest.java | 6 +-
.../IgniteStandByClusterTest.java | 6 +-
.../extended/GridActivateExtensionTest.java | 4 +-
...gniteAbstractStandByClientReconnectTest.java | 140 +-
.../IgniteStandByClientReconnectTest.java | 94 +-
...eStandByClientReconnectToNewClusterTest.java | 6 +-
.../wal/SegmentedRingByteBufferTest.java | 188 +-
.../GridCacheQueryTransformerSelfTest.java | 79 +
...CacheContinuousQueryCounterAbstractTest.java | 4 +-
...acheContinuousQueryRandomOperationsTest.java | 156 +-
...ContinuousWithTransformerClientSelfTest.java | 40 +
...heContinuousWithTransformerFailoverTest.java | 309 ++
...eContinuousWithTransformerLocalSelfTest.java | 29 +
...nuousWithTransformerPartitionedSelfTest.java | 29 +
...uousWithTransformerRandomOperationsTest.java | 31 +
...inuousWithTransformerReplicatedSelfTest.java | 511 +++
...ridCacheContinuousQueryAbstractSelfTest.java | 10 +-
...eContinuousQueryMultiNodesFilteringTest.java | 3 +-
.../TxDeadlockDetectionNoHangsTest.java | 2 +
.../TxPessimisticDeadlockDetectionTest.java | 2 +
.../transactions/TxRollbackOnTimeoutTest.java | 10 +-
.../processors/database/BPlusTreeSelfTest.java | 12 +-
.../database/CacheFreeListImplSelfTest.java | 565 +++
.../database/FreeListImplSelfTest.java | 565 ---
.../database/IgniteDbAbstractTest.java | 29 +-
.../IgniteDbClientNearCachePutGetTest.java | 39 -
.../database/IgniteDbDynamicCacheSelfTest.java | 9 +-
.../IgniteDbMemoryLeakAbstractTest.java | 2 +-
.../database/IgniteDbPutGetAbstractTest.java | 1 +
.../IgniteDbPutGetWithCacheStoreTest.java | 196 +
.../database/IndexStorageSelfTest.java | 171 +
.../database/MetadataStorageSelfTest.java | 171 -
.../database/SwapPathConstructionSelfTest.java | 4 +-
.../datastreamer/DataStreamerImplSelfTest.java | 22 +-
.../IgniteDataStreamerPerformanceTest.java | 12 +-
.../processors/igfs/IgfsSizeSelfTest.java | 19 +-
.../cache/GridCacheCommandHandlerSelfTest.java | 9 +-
.../processors/service/DummyService.java | 12 +-
...gniteServiceConfigVariationsFullApiTest.java | 58 +-
.../internal/sql/SqlParserBulkLoadSelfTest.java | 90 +
.../sql/SqlParserSetStreamingSelfTest.java | 134 +
.../internal/util/IgniteDevOnlyLogTest.java | 113 +
.../util/future/GridCompoundFutureSelfTest.java | 8 +-
.../future/GridFutureListenPerformanceTest.java | 8 +-
.../util/future/GridFutureQueueTest.java | 5 +-
.../ipc/shmem/IpcSharedMemorySpaceSelfTest.java | 6 +-
.../offheap/GridOffHeapMapAbstractSelfTest.java | 4 +-
...idOffHeapPartitionedMapAbstractSelfTest.java | 5 +-
...apPartitionedMapPerformanceAbstractTest.java | 6 +-
.../ignite/jvmtest/BlockingQueueTest.java | 4 +-
.../ignite/jvmtest/ConcurrentMapTest.java | 200 -
.../ignite/lang/GridBasicPerformanceTest.java | 14 +-
.../lang/GridFutureListenPerformanceTest.java | 8 +-
.../GridCircularBufferPerformanceTest.java | 15 +-
.../lang/utils/GridCircularBufferSelfTest.java | 9 +-
.../utils/GridConcurrentHashMapSelfTest.java | 110 -
.../colocation/GridTestCacheStore.java | 4 +-
.../communication/GridIoManagerBenchmark.java | 14 +-
.../communication/GridIoManagerBenchmark0.java | 14 +-
.../GridContinuousOperationsLoadTest.java | 4 +-
.../ignite/loadtests/dsi/GridDsiPerfJob.java | 6 +-
...dBoundedConcurrentLinkedHashSetLoadTest.java | 4 +-
.../loadtests/hashmap/GridCacheTestContext.java | 2 +
.../job/GridJobExecutionLoadTestClient.java | 4 +-
...GridJobExecutionLoadTestClientSemaphore.java | 4 +-
...JobExecutionSingleNodeSemaphoreLoadTest.java | 6 +-
...apPartitionedMapPerformanceAbstractTest.java | 6 +-
.../marshaller/MarshallerContextSelfTest.java | 10 +-
.../marshaller/MarshallerContextTestImpl.java | 6 +-
.../messaging/IgniteMessagingSendAsyncTest.java | 4 +-
.../ignite/p2p/P2PStreamingClassLoaderTest.java | 121 +
.../platform/PlatformCacheWriteMetricsTask.java | 5 +
.../tcp/GridTcpCommunicationSpiLanTest.java | 13 +-
...cpCommunicationSpiMultithreadedSelfTest.java | 30 +-
.../tcp/TcpCommunicationStatisticsTest.java | 14 +-
.../IgniteDiscoveryCacheReuseSelfTest.java | 18 +-
.../DiscoveryUnmarshalVulnerabilityTest.java | 180 +
.../tcp/TcpDiscoverySpiReconnectDelayTest.java | 446 ++
.../startup/GridRandomCommandLineLoader.java | 2 +-
...ocketStreamerUnmarshalVulnerabilityTest.java | 222 +
.../testframework/GridSpiTestContext.java | 6 +
.../ignite/testframework/GridTestUtils.java | 13 +-
.../junits/GridTestKernalContext.java | 3 +-
.../junits/common/GridCommonAbstractTest.java | 125 +-
.../multijvm/IgniteClusterProcessProxy.java | 41 +
.../ignite/testsuites/IgniteBasicTestSuite.java | 14 +-
.../IgniteCacheEvictionSelfTestSuite.java | 3 +
.../IgniteCacheFailoverTestSuite.java | 4 +
.../IgniteCacheFailoverTestSuite2.java | 4 +
.../IgniteCacheMetricsSelfTestSuite.java | 4 +-
.../ignite/testsuites/IgniteCacheTestSuite.java | 2 +
.../testsuites/IgniteCacheTestSuite2.java | 6 +
.../testsuites/IgniteCacheTestSuite3.java | 6 +
.../testsuites/IgniteCacheTestSuite6.java | 7 +
.../ignite/testsuites/IgniteIpcTestSuite.java | 38 +
.../testsuites/IgniteP2PSelfTestSuite.java | 2 +
.../ignite/testsuites/IgnitePdsTestSuite.java | 66 +-
.../ignite/testsuites/IgnitePdsTestSuite2.java | 58 +-
.../testsuites/IgnitePdsUnitTestSuite.java | 38 +
.../testsuites/IgniteReproducingSuite.java | 6 +-
.../IgniteSpiDiscoverySelfTestSuite.java | 6 +
.../testsuites/IgniteStandByClusterSuite.java | 5 +
.../testsuites/IgniteStreamSelfTestSuite.java | 2 +
.../testsuites/IgniteUtilSelfTestSuite.java | 10 +-
.../ignite/util/GridCommandHandlerTest.java | 276 ++
.../util/mbeans/GridMBeanDisableSelfTest.java | 69 +-
.../mbeans/GridMBeanExoticNamesSelfTest.java | 70 +
modules/dev-utils/pom.xml | 28 +-
.../development/utils/IgniteWalConverter.java | 32 +-
.../development/utils/RecordSizeCountStat.java | 83 +
.../ignite/development/utils/TxWalStat.java | 318 ++
.../ignite/development/utils/WalStat.java | 328 ++
modules/direct-io/licenses/apache-2.0.txt | 202 +
modules/direct-io/pom.xml | 122 +
.../cache/persistence/file/AlignedBuffers.java | 72 +
.../file/AlignedBuffersDirectFileIO.java | 511 +++
.../file/AlignedBuffersDirectFileIOFactory.java | 177 +
.../persistence/file/IgniteNativeIoLib.java | 405 ++
.../persistence/file/LinuxNativeIoPlugin.java | 25 +
.../file/LinuxNativeIoPluginProvider.java | 239 ++
.../org.apache.ignite.plugin.PluginProvider | 1 +
.../IgniteNativeIoWithNoPersistenceTest.java | 76 +
.../testsuites/IgnitePdsNativeIoTestSuite.java | 41 +
.../testsuites/IgnitePdsNativeIoTestSuite2.java | 36 +
.../IgnitePdsReplacementNativeIoTest.java | 44 +
modules/extdata/p2p/pom.xml | 2 +-
.../tests/p2p/NoopCacheEntryProcessor.java | 36 +
modules/extdata/platform/pom.xml | 2 +-
.../extdata/uri/modules/uri-dependency/pom.xml | 2 +-
modules/extdata/uri/pom.xml | 2 +-
modules/flink/pom.xml | 2 +-
modules/flume/pom.xml | 2 +-
modules/gce/pom.xml | 2 +-
modules/geospatial/pom.xml | 2 +-
modules/hadoop/pom.xml | 4 +-
.../hadoop/counter/HadoopCountersImpl.java | 4 +-
.../hadoop/impl/fs/HadoopLazyConcurrentMap.java | 4 +-
.../hadoop/impl/igfs/HadoopIgfsIpcIo.java | 6 +-
.../hadoop/impl/igfs/HadoopIgfsOutProc.java | 5 +-
.../processors/hadoop/impl/v2/HadoopV2Job.java | 4 +-
.../hadoop/jobtracker/HadoopJobTracker.java | 8 +-
.../taskexecutor/HadoopExecutorService.java | 4 +-
.../external/HadoopExternalTaskExecutor.java | 10 +-
.../HadoopExternalCommunication.java | 4 +-
.../processors/hadoop/HadoopSharedMap.java | 6 +-
.../hadoop/impl/HadoopAbstractSelfTest.java | 15 +-
.../hadoop/impl/HadoopCommandLineTest.java | 4 +-
.../HadoopIgfs20FileSystemAbstractSelfTest.java | 4 +-
.../IgniteHadoopFileSystemAbstractSelfTest.java | 4 +-
.../taskexecutor/HadoopExecutorServiceTest.java | 6 +-
modules/hibernate-4.2/pom.xml | 2 +-
modules/hibernate-5.1/pom.xml | 2 +-
modules/hibernate-core/pom.xml | 2 +-
modules/ignored-tests/pom.xml | 2 +-
modules/indexing/pom.xml | 50 +-
.../query/h2/DmlStatementsProcessor.java | 554 ++-
.../query/h2/H2CachedStatementKey.java | 108 +
.../processors/query/h2/H2DmlPlanKey.java | 81 -
.../processors/query/h2/H2StatementCache.java | 28 +-
.../internal/processors/query/h2/H2Utils.java | 28 +-
.../processors/query/h2/IgniteH2Indexing.java | 773 ++--
.../processors/query/h2/ParsingResult.java | 102 +
.../query/h2/database/H2TreeIndex.java | 15 +
.../query/h2/ddl/DdlStatementsProcessor.java | 79 +-
.../processors/query/h2/dml/DmlBatchSender.java | 247 +-
.../processors/query/h2/dml/DmlUtils.java | 12 +
.../processors/query/h2/dml/UpdateMode.java | 11 +-
.../processors/query/h2/dml/UpdatePlan.java | 83 +-
.../query/h2/dml/UpdatePlanBuilder.java | 101 +-
.../query/h2/opt/GridH2IndexBase.java | 10 +
.../query/h2/opt/GridH2MetaTable.java | 6 +-
.../query/h2/opt/GridH2QueryContext.java | 4 +-
.../processors/query/h2/opt/GridH2Table.java | 15 +-
.../query/h2/sql/GridSqlQueryParser.java | 93 +-
.../query/h2/sql/GridSqlQuerySplitter.java | 3 +-
.../query/h2/twostep/GridMapQueryExecutor.java | 16 +-
.../h2/twostep/GridReduceQueryExecutor.java | 5 +-
.../query/h2/twostep/MapNodeResults.java | 6 +-
.../query/h2/twostep/MapQueryLazyWorker.java | 11 +-
.../query/h2/twostep/MapQueryResult.java | 24 +-
.../query/h2/twostep/MapQueryResults.java | 18 +-
.../cache/GridCacheCrossCacheQuerySelfTest.java | 90 +-
...GridCacheLazyQueryPartitionsReleaseTest.java | 9 +-
.../cache/GridCacheQuerySimpleBenchmark.java | 6 +-
.../IgniteBinaryObjectFieldsQuerySelfTest.java | 7 +-
.../cache/IgniteCacheAbstractQuerySelfTest.java | 59 +-
...eLockPartitionOnAffinityRunAbstractTest.java | 8 +-
...PartitionOnAffinityRunAtomicCacheOpTest.java | 18 +-
...niteCacheLockPartitionOnAffinityRunTest.java | 29 +-
.../cache/IgniteCacheSqlQueryErrorSelfTest.java | 214 +
...CheckClusterStateBeforeExecuteQueryTest.java | 8 +-
.../cache/IgniteErrorOnRebalanceTest.java | 2 +-
...leNodeWithIndexingPutGetPersistenceTest.java | 7 +-
.../cache/SqlFieldsQuerySelfTest.java | 53 +
...lientQueryReplicatedNodeRestartSelfTest.java | 1 +
...stributedPartitionQueryAbstractSelfTest.java | 4 +-
...chePartitionedQueryEvtsDisabledSelfTest.java | 30 +
.../IgniteCacheQueryNodeRestartSelfTest.java | 56 +-
...acheReplicatedQueryEvtsDisabledSelfTest.java | 30 +
.../IgniteCacheReplicatedQuerySelfTest.java | 54 +-
.../cache/index/AbstractSchemaSelfTest.java | 3 +
.../cache/index/DynamicColumnsAbstractTest.java | 2 +-
.../DynamicIndexAbstractBasicSelfTest.java | 29 +-
.../index/DynamicIndexAbstractSelfTest.java | 33 +-
.../H2DynamicColumnsAbstractBasicSelfTest.java | 37 +
.../index/H2DynamicIndexingComplexTest.java | 4 +-
.../cache/index/H2DynamicTableSelfTest.java | 14 +-
.../cache/index/LongIndexNameTest.java | 14 +-
.../index/OptimizedMarshallerIndexNameTest.java | 2 +-
.../local/IgniteCacheLocalQuerySelfTest.java | 62 +-
...onnectorConfigurationValidationSelfTest.java | 66 +
.../client/IgniteDataStreamerTest.java | 118 +
...eDbSingleNodeWithIndexingWalRestoreTest.java | 7 +-
...oreQueryWithMultipleClassesPerCacheTest.java | 13 +-
.../IgnitePersistentStoreSchemaLoadTest.java | 25 +-
...ckPartitionOnAffinityRunAtomicCacheTest.java | 96 +
...neLockPartitionOnAffinityRunTxCacheTest.java | 87 +
...ngBaselineCacheQueryNodeRestartSelfTest.java | 124 +
...StableBaselineBinObjFieldsQuerySelfTest.java | 77 +
...eBaselineCacheQueryNodeRestartsSelfTest.java | 68 +
.../query/IgniteCachelessQueriesSelfTest.java | 420 ++
.../query/IgniteQueryDedicatedPoolTest.java | 15 +-
.../query/IgniteSqlDefaultValueTest.java | 2 +-
.../query/IgniteSqlNotNullConstraintTest.java | 2 +-
.../query/IgniteSqlSchemaIndexingTest.java | 11 +-
...IgniteSqlSkipReducerOnUpdateDmlSelfTest.java | 4 +-
.../MultipleStatementsSqlQuerySelfTest.java | 8 +-
.../query/SqlPushDownFunctionTest.java | 78 +
.../processors/query/SqlSchemaSelfTest.java | 12 +-
.../IgniteCacheAffinityRunTestSuite.java | 4 +
.../IgniteCacheQuerySelfTestSuite.java | 28 +-
.../IgniteCacheQuerySelfTestSuite2.java | 4 +
.../IgniteCacheQuerySelfTestSuite3.java | 13 +
.../IgniteCacheWithIndexingTestSuite.java | 3 +
.../IgnitePdsWithIndexingCoreTestSuite.java | 4 +
modules/jcl/pom.xml | 2 +-
modules/jms11/pom.xml | 2 +-
modules/jta/pom.xml | 2 +-
modules/kafka/pom.xml | 2 +-
modules/kubernetes/pom.xml | 2 +-
modules/log4j/pom.xml | 2 +-
.../apache/ignite/logger/log4j/Log4JLogger.java | 100 +-
modules/log4j/src/test/config/log4j-debug.xml | 50 +
modules/log4j/src/test/config/log4j-info.xml | 50 +
.../logger/log4j/GridLog4jConfigUpdateTest.java | 163 +
.../logger/log4j/GridLog4jWatchDelayTest.java | 49 +
.../ignite/testsuites/IgniteLog4jTestSuite.java | 35 +-
modules/log4j2/pom.xml | 6 +-
.../ignite/logger/log4j2/Log4J2Logger.java | 45 +-
modules/log4j2/src/test/config/log4j2-debug.xml | 37 +
modules/log4j2/src/test/config/log4j2-info.xml | 37 +
.../log4j2/src/test/config/log4j2-markers.xml | 38 +
.../logger/log4j2/Log4j2ConfigUpdateTest.java | 152 +
.../logger/log4j2/Log4j2LoggerMarkerTest.java | 110 +
.../testsuites/IgniteLog4j2TestSuite.java | 8 +-
modules/mesos/pom.xml | 13 +-
modules/ml/README.txt | 16 +-
modules/ml/pom.xml | 40 +-
.../org/apache/ignite/ml/DatasetTrainer.java | 42 +
.../main/java/org/apache/ignite/ml/Trainer.java | 2 +
.../org/apache/ignite/ml/dataset/Dataset.java | 213 +
.../ignite/ml/dataset/DatasetBuilder.java | 49 +
.../ignite/ml/dataset/DatasetFactory.java | 387 ++
.../ml/dataset/PartitionContextBuilder.java | 58 +
.../ignite/ml/dataset/PartitionDataBuilder.java | 63 +
.../apache/ignite/ml/dataset/UpstreamEntry.java | 53 +
.../dataset/impl/cache/CacheBasedDataset.java | 168 +
.../impl/cache/CacheBasedDatasetBuilder.java | 95 +
.../ml/dataset/impl/cache/package-info.java | 22 +
.../dataset/impl/cache/util/ComputeUtils.java | 251 ++
.../util/DatasetAffinityFunctionWrapper.java | 75 +
.../impl/cache/util/PartitionDataStorage.java | 65 +
.../impl/cache/util/UpstreamCursorAdapter.java | 68 +
.../dataset/impl/cache/util/package-info.java | 22 +
.../ml/dataset/impl/local/LocalDataset.java | 88 +
.../dataset/impl/local/LocalDatasetBuilder.java | 137 +
.../ml/dataset/impl/local/package-info.java | 22 +
.../ignite/ml/dataset/impl/package-info.java | 22 +
.../apache/ignite/ml/dataset/package-info.java | 22 +
.../ml/dataset/primitive/DatasetWrapper.java | 63 +
.../ml/dataset/primitive/SimpleDataset.java | 216 +
.../dataset/primitive/SimpleLabeledDataset.java | 39 +
.../builder/context/EmptyContextBuilder.java | 39 +
.../primitive/builder/context/package-info.java | 22 +
.../builder/data/SimpleDatasetDataBuilder.java | 76 +
.../data/SimpleLabeledDatasetDataBuilder.java | 86 +
.../primitive/builder/data/package-info.java | 22 +
.../dataset/primitive/builder/package-info.java | 22 +
.../dataset/primitive/context/EmptyContext.java | 28 +
.../dataset/primitive/context/package-info.java | 22 +
.../primitive/data/SimpleDatasetData.java | 69 +
.../data/SimpleLabeledDatasetData.java | 79 +
.../ml/dataset/primitive/data/package-info.java | 22 +
.../ml/dataset/primitive/package-info.java | 28 +
.../apache/ignite/ml/genetic/Chromosome.java | 107 +
.../apache/ignite/ml/genetic/CrossOverJob.java | 142 +
.../apache/ignite/ml/genetic/CrossOverTask.java | 137 +
.../apache/ignite/ml/genetic/FitnessJob.java | 101 +
.../apache/ignite/ml/genetic/FitnessTask.java | 103 +
.../org/apache/ignite/ml/genetic/GAGrid.java | 454 +++
.../java/org/apache/ignite/ml/genetic/Gene.java | 86 +
.../ignite/ml/genetic/IFitnessFunction.java | 39 +
.../org/apache/ignite/ml/genetic/MutateJob.java | 94 +
.../apache/ignite/ml/genetic/MutateTask.java | 191 +
.../ignite/ml/genetic/TruncateSelectionJob.java | 88 +
.../ml/genetic/TruncateSelectionTask.java | 173 +
.../ml/genetic/cache/GeneCacheConfig.java | 52 +
.../ml/genetic/cache/PopulationCacheConfig.java | 53 +
.../ignite/ml/genetic/cache/package-info.java | 21 +
.../ml/genetic/functions/GAGridFunction.java | 132 +
.../ml/genetic/functions/package-info.java | 21 +
.../apache/ignite/ml/genetic/package-info.java | 21 +
.../genetic/parameter/ChromosomeCriteria.java | 50 +
.../ml/genetic/parameter/GAConfiguration.java | 333 ++
.../ml/genetic/parameter/GAGridConstants.java | 37 +
.../genetic/parameter/ITerminateCriteria.java | 39 +
.../ml/genetic/parameter/package-info.java | 21 +
.../ignite/ml/genetic/utils/GAGridUtils.java | 124 +
.../ignite/ml/genetic/utils/package-info.java | 21 +
.../ml/math/isolve/IterativeSolverResult.java | 64 +
.../LinSysPartitionDataBuilderOnHeap.java | 85 +
.../math/isolve/LinSysPartitionDataOnHeap.java | 75 +
.../ml/math/isolve/lsqr/AbstractLSQR.java | 333 ++
.../ignite/ml/math/isolve/lsqr/LSQROnHeap.java | 102 +
.../math/isolve/lsqr/LSQRPartitionContext.java | 41 +
.../ignite/ml/math/isolve/lsqr/LSQRResult.java | 140 +
.../ml/math/isolve/lsqr/package-info.java | 22 +
.../ignite/ml/math/isolve/package-info.java | 22 +
.../ignite/ml/nn/LocalBatchTrainerInput.java | 42 -
.../org/apache/ignite/ml/nn/LossFunctions.java | 47 -
.../ml/nn/MLPGroupUpdateTrainerCacheInput.java | 33 +-
.../ignite/ml/nn/MultilayerPerceptron.java | 2 +-
.../AbstractMLPGroupUpdateTrainerInput.java | 2 +-
.../distributed/MLPGroupUpdateTrainer.java | 91 +-
.../MLPGroupUpdateTrainerDataCache.java | 2 +-
.../distributed/MLPGroupUpdateTrainingData.java | 35 +-
.../MLPGroupUpdateTrainingLoopData.java | 36 +-
.../trainers/distributed/MLPMetaoptimizer.java | 5 +-
.../nn/trainers/distributed/package-info.java | 22 +
.../ml/nn/trainers/local/LocalBatchTrainer.java | 179 -
.../nn/trainers/local/MLPLocalBatchTrainer.java | 13 +-
.../ml/nn/updaters/BaseSmoothParametrized.java | 66 -
.../ml/nn/updaters/NesterovParameterUpdate.java | 94 -
.../nn/updaters/NesterovUpdateCalculator.java | 85 -
.../nn/updaters/ParameterUpdateCalculator.java | 58 -
.../ml/nn/updaters/RPropParameterUpdate.java | 228 --
.../ml/nn/updaters/RPropUpdateCalculator.java | 151 -
.../ml/nn/updaters/SimpleGDParameter.java | 77 -
.../nn/updaters/SimpleGDUpdateCalculator.java | 66 -
.../ml/nn/updaters/SmoothParametrized.java | 27 -
.../ignite/ml/nn/updaters/package-info.java | 22 -
.../ml/optimization/BarzilaiBorweinUpdater.java | 8 +-
.../ml/optimization/BaseParametrized.java | 58 +
.../ignite/ml/optimization/GradientDescent.java | 3 +-
.../ml/optimization/GradientFunction.java | 2 +-
.../ignite/ml/optimization/LossFunctions.java | 47 +
.../ignite/ml/optimization/Parametrized.java | 26 +
.../ignite/ml/optimization/SimpleUpdater.java | 3 +-
.../ml/optimization/SmoothParametrized.java | 45 +
.../NesterovParameterUpdate.java | 94 +
.../NesterovUpdateCalculator.java | 86 +
.../ParameterUpdateCalculator.java | 59 +
.../updatecalculators/RPropParameterUpdate.java | 228 ++
.../RPropUpdateCalculator.java | 152 +
.../SimpleGDParameterUpdate.java | 89 +
.../SimpleGDUpdateCalculator.java | 81 +
.../updatecalculators/package-info.java | 22 +
.../util/SparseDistributedMatrixMapReducer.java | 2 +-
.../ml/preprocessing/PreprocessingTrainer.java | 42 +
.../NormalizationPartitionData.java | 58 +
.../NormalizationPreprocessor.java | 88 +
.../normalization/NormalizationTrainer.java | 90 +
.../normalization/package-info.java | 22 +
.../ignite/ml/preprocessing/package-info.java | 22 +
.../linear/LinearRegressionLSQRTrainer.java | 70 +
.../apache/ignite/ml/structures/Dataset.java | 31 +-
.../apache/ignite/ml/structures/DatasetRow.java | 1 -
.../ignite/ml/structures/LabeledDataset.java | 43 +-
.../structures/LabeledDatasetTestTrainPair.java | 4 +-
.../preprocessing/LabeledDatasetLoader.java | 31 +-
.../preprocessing/LabellingMachine.java | 10 +-
.../ml/structures/preprocessing/Normalizer.java | 2 +
.../svm/SVMLinearBinaryClassificationModel.java | 186 +
.../SVMLinearBinaryClassificationTrainer.java | 264 ++
.../ignite/ml/svm/SVMPartitionContext.java | 28 +
.../ml/svm/SVMPartitionDataBuilderOnHeap.java | 86 +
.../org/apache/ignite/ml/svm/package-info.java | 22 +
.../org/apache/ignite/ml/trainers/Trainer.java | 5 +-
.../trainers/group/BaseLocalProcessorJob.java | 3 +-
.../group/GroupTrainerBaseProcessorTask.java | 3 +-
.../group/MetaoptimizerDistributedStep.java | 13 +-
.../ml/trainers/group/ResultAndUpdates.java | 3 +-
.../ml/trainers/group/UpdateStrategies.java | 47 +
.../ml/trainers/group/UpdatesStrategy.java | 94 +
.../trainers/group/chain/ComputationsChain.java | 4 +-
.../ml/trainers/local/LocalBatchTrainer.java | 178 +
.../trainers/local/LocalBatchTrainerInput.java | 42 +
.../ignite/ml/trainers/local/package-info.java | 22 +
.../java/org/apache/ignite/ml/util/Utils.java | 24 +-
.../org/apache/ignite/ml/IgniteMLTestSuite.java | 14 +-
.../org/apache/ignite/ml/LocalModelsTest.java | 19 +
.../ignite/ml/dataset/DatasetTestSuite.java | 45 +
.../cache/CacheBasedDatasetBuilderTest.java | 107 +
.../impl/cache/CacheBasedDatasetTest.java | 351 ++
.../impl/cache/util/ComputeUtilsTest.java | 309 ++
.../DatasetAffinityFunctionWrapperTest.java | 110 +
.../cache/util/PartitionDataStorageTest.java | 49 +
.../impl/local/LocalDatasetBuilderTest.java | 91 +
.../dataset/primitive/DatasetWrapperTest.java | 87 +
.../ml/genetic/GAGridCalculateFitnessTest.java | 141 +
.../genetic/GAGridInitializePopulationTest.java | 153 +
.../ignite/ml/genetic/GAGridTestSuite.java | 33 +
.../ml/genetic/PasswordFitnessFunction.java | 61 +
.../ignite/ml/knn/LabeledDatasetTest.java | 1 +
.../apache/ignite/ml/math/ExternalizeTest.java | 1 +
.../ignite/ml/math/MathImplLocalTestSuite.java | 4 +-
.../org/apache/ignite/ml/math/TracerTest.java | 37 +-
.../ml/math/isolve/lsqr/LSQROnHeapTest.java | 134 +
.../ignite/ml/nn/MLPGroupTrainerTest.java | 38 +-
.../ignite/ml/nn/MLPLocalTrainerTest.java | 17 +-
.../java/org/apache/ignite/ml/nn/MLPTest.java | 1 +
.../ml/nn/SimpleMLPLocalBatchTrainerInput.java | 1 +
.../ml/nn/performance/MnistDistributed.java | 8 +-
.../ignite/ml/nn/performance/MnistLocal.java | 8 +-
.../ml/optimization/OptimizationTestSuite.java | 33 +
.../preprocessing/PreprocessingTestSuite.java | 35 +
.../NormalizationPreprocessorTest.java | 54 +
.../normalization/NormalizationTrainerTest.java | 76 +
.../ml/regressions/RegressionsTestSuite.java | 4 +-
.../linear/LinearRegressionLSQRTrainerTest.java | 124 +
.../ignite/ml/svm/SVMBinaryTrainerTest.java | 74 +
.../org/apache/ignite/ml/svm/SVMModelTest.java | 113 +
.../org/apache/ignite/ml/svm/SVMTestSuite.java | 33 +
.../ml/trainers/group/TestGroupTrainer.java | 2 +-
.../ml/src/test/resources/datasets/README.md | 5 +-
modules/mqtt/pom.xml | 2 +-
modules/osgi-karaf/pom.xml | 2 +-
modules/osgi-paxlogging/pom.xml | 2 +-
modules/osgi/pom.xml | 2 +-
.../org/apache/ignite/osgi/IgniteOsgiUtils.java | 4 +-
.../classloaders/ContainerSweepClassLoader.java | 4 +-
modules/platforms/cpp/common/configure.ac | 2 +-
.../include/ignite/common/platform_utils.h | 17 +
.../cpp/common/include/ignite/common/utils.h | 7 +
.../os/linux/src/common/platform_utils.cpp | 15 +
.../os/win/src/common/dynamic_load_os.cpp | 4 +
.../common/os/win/src/common/platform_utils.cpp | 15 +
.../platforms/cpp/common/src/common/utils.cpp | 9 +
modules/platforms/cpp/configure.ac | 2 +-
modules/platforms/cpp/configure.acrel | 2 +-
.../cpp/core-test/config/cache-identity-32.xml | 5 +-
.../cpp/core-test/config/cache-query-32.xml | 6 +-
.../config/cache-query-continuous-32.xml | 6 +-
.../cpp/core-test/config/cache-store-32.xml | 4 +-
.../cpp/core-test/config/cache-test-32.xml | 6 +-
.../cpp/core-test/config/isolated-32.xml | 5 +-
.../core-test/config/persistence-store-32.xml | 5 +-
modules/platforms/cpp/core-test/configure.ac | 2 +-
.../cpp/core-test/src/cache_query_test.cpp | 58 +-
modules/platforms/cpp/core/configure.ac | 2 +-
modules/platforms/cpp/examples/configure.ac | 2 +-
modules/platforms/cpp/ignite/configure.ac | 2 +-
modules/platforms/cpp/odbc-test/Makefile.am | 6 +-
.../cpp/odbc-test/config/queries-default.xml | 10 +-
.../cpp/odbc-test/config/queries-ssl-32.xml | 77 +
.../cpp/odbc-test/config/queries-ssl.xml | 59 +
.../cpp/odbc-test/config/queries-test-32.xml | 7 +-
.../odbc-test/config/queries-test-default.xml | 39 -
.../odbc-test/config/queries-test-noodbc-32.xml | 53 -
.../odbc-test/config/queries-test-noodbc.xml | 35 -
.../cpp/odbc-test/config/queries-test.xml | 3 +-
.../platforms/cpp/odbc-test/config/ssl/ca.pem | 24 +
.../cpp/odbc-test/config/ssl/client_full.pem | 84 +
.../cpp/odbc-test/config/ssl/client_unknown.pem | 50 +
.../cpp/odbc-test/config/ssl/server.jks | Bin 0 -> 4256 bytes
.../cpp/odbc-test/config/ssl/trust.jks | Bin 0 -> 1089 bytes
.../cpp/odbc-test/include/odbc_test_suite.h | 8 +-
.../cpp/odbc-test/include/test_utils.h | 5 +
.../cpp/odbc-test/project/vs/odbc-test.vcxproj | 15 +-
.../project/vs/odbc-test.vcxproj.filters | 14 +-
.../cpp/odbc-test/src/api_robustness_test.cpp | 101 +-
.../src/application_data_buffer_test.cpp | 4 +-
.../platforms/cpp/odbc-test/src/column_test.cpp | 174 +-
.../platforms/cpp/odbc-test/src/errors_test.cpp | 90 +-
.../cpp/odbc-test/src/meta_queries_test.cpp | 96 +-
.../cpp/odbc-test/src/odbc_test_suite.cpp | 67 +-
.../cpp/odbc-test/src/queries_ssl_test.cpp | 283 ++
.../cpp/odbc-test/src/queries_test.cpp | 18 +-
.../platforms/cpp/odbc-test/src/row_test.cpp | 1 +
.../cpp/odbc-test/src/sql_get_info_test.cpp | 8 +-
.../odbc-test/src/sql_test_suite_fixture.cpp | 4 +
.../platforms/cpp/odbc-test/src/test_utils.cpp | 9 +
modules/platforms/cpp/odbc/Makefile.am | 5 +-
modules/platforms/cpp/odbc/include/Makefile.am | 6 +-
.../ignite/odbc/app/application_data_buffer.h | 104 +-
.../include/ignite/odbc/app/parameter_set.h | 26 +-
.../cpp/odbc/include/ignite/odbc/column.h | 3 +-
.../cpp/odbc/include/ignite/odbc/common_types.h | 6 +
.../include/ignite/odbc/config/configuration.h | 110 +-
.../cpp/odbc/include/ignite/odbc/connection.h | 9 +-
.../cpp/odbc/include/ignite/odbc/message.h | 12 -
.../odbc/include/ignite/odbc/query/data_query.h | 13 +-
.../cpp/odbc/include/ignite/odbc/row.h | 7 +-
.../odbc/include/ignite/odbc/socket_client.h | 109 +
.../ignite/odbc/ssl/secure_socket_client.h | 148 +
.../odbc/include/ignite/odbc/ssl/ssl_bindings.h | 303 ++
.../odbc/include/ignite/odbc/ssl/ssl_gateway.h | 151 +
.../cpp/odbc/include/ignite/odbc/ssl/ssl_mode.h | 55 +
.../cpp/odbc/include/ignite/odbc/statement.h | 3 +
.../include/ignite/odbc/system/socket_client.h | 151 -
.../ignite/odbc/system/tcp_socket_client.h | 151 +
.../odbc/system/ui/dsn_configuration_window.h | 113 +-
.../cpp/odbc/include/ignite/odbc/type_traits.h | 6 +-
.../cpp/odbc/install/ignite-odbc-amd64.wxs | 2 +-
.../cpp/odbc/install/ignite-odbc-x86.wxs | 2 +-
.../odbc/os/linux/src/system/socket_client.cpp | 360 --
.../os/linux/src/system/tcp_socket_client.cpp | 384 ++
.../odbc/os/win/src/system/socket_client.cpp | 432 --
.../os/win/src/system/tcp_socket_client.cpp | 462 +++
.../src/system/ui/dsn_configuration_window.cpp | 315 +-
.../cpp/odbc/os/win/src/system/ui/window.cpp | 9 +-
.../platforms/cpp/odbc/project/vs/odbc.vcxproj | 36 +-
.../cpp/odbc/project/vs/odbc.vcxproj.filters | 39 +-
.../odbc/src/app/application_data_buffer.cpp | 545 ++-
.../cpp/odbc/src/app/parameter_set.cpp | 21 +-
modules/platforms/cpp/odbc/src/column.cpp | 55 +-
.../cpp/odbc/src/config/configuration.cpp | 8 +
.../cpp/odbc/src/config/connection_info.cpp | 139 +-
modules/platforms/cpp/odbc/src/connection.cpp | 74 +-
.../odbc/src/diagnostic/diagnosable_adapter.cpp | 6 +-
.../odbc/src/diagnostic/diagnostic_record.cpp | 12 +
modules/platforms/cpp/odbc/src/dsn_config.cpp | 24 +-
modules/platforms/cpp/odbc/src/message.cpp | 4 +-
.../platforms/cpp/odbc/src/meta/column_meta.cpp | 37 +-
modules/platforms/cpp/odbc/src/odbc.cpp | 16 +-
.../cpp/odbc/src/query/batch_query.cpp | 13 +-
.../platforms/cpp/odbc/src/query/data_query.cpp | 77 +-
modules/platforms/cpp/odbc/src/row.cpp | 9 +-
.../cpp/odbc/src/ssl/secure_socket_client.cpp | 419 ++
.../platforms/cpp/odbc/src/ssl/ssl_gateway.cpp | 210 +
modules/platforms/cpp/odbc/src/ssl/ssl_mode.cpp | 43 +
modules/platforms/cpp/odbc/src/statement.cpp | 35 +
modules/platforms/cpp/odbc/src/type_traits.cpp | 50 +-
.../Properties/AssemblyInfo.cs | 8 +-
.../Apache.Ignite.AspNet.nuspec | 2 +-
.../Properties/AssemblyInfo.cs | 8 +-
.../Apache.Ignite.Benchmarks.csproj | 5 +
.../Apache.Ignite.Benchmarks/BenchmarkRunner.cs | 12 +-
.../Interop/GetAllBenchmark.cs | 71 +
.../Interop/GetAllBinaryBenchmark.cs | 58 +
.../Apache.Ignite.Benchmarks/Model/Doubles.cs | 69 +
.../Properties/AssemblyInfo.cs | 8 +-
.../ThinClient/ThinClientGetAllBenchmark.cs | 72 +
.../ThinClientGetAllBinaryBenchmark.cs | 58 +
.../Apache.Ignite.Core.Tests.DotNetCore.csproj | 28 +-
.../Common/TestUtils.DotNetCore.cs | 3 +-
.../Properties/AssemblyInfo.cs | 8 +-
.../Properties/AssemblyInfo.cs | 8 +-
.../Apache.Ignite.Core.Tests.csproj | 56 +-
.../ApiParity/CacheConfigurationParityTest.cs | 3 +-
.../ClientConnectorConfigurationParityTest.cs | 11 +-
.../ApiParity/ClusterNodeParityTest.cs | 45 +
.../ApiParity/DataRegionMetricsParityTest.cs | 10 +-
.../DataStorageConfigurationParityTest.cs | 2 +-
.../ApiParity/DataStorageMetricsParityTest.cs | 2 +-
.../ApiParity/IgniteConfigurationParityTest.cs | 3 +-
.../ApiParity/IgniteParityTest.cs | 2 -
.../ApiParity/ParityTest.cs | 26 +-
.../QueryEntityConfigurationParityTest.cs | 2 +-
.../Binary/IO/BinaryStreamsTest.cs | 5 +-
.../Affinity/AffinityFunctionSpringTest.cs | 2 +-
.../Cache/CacheAbstractTest.cs | 72 -
.../Cache/CacheAbstractTransactionalTest.cs | 5 +
.../Cache/CacheConfigurationTest.cs | 2 +-
.../Cache/CacheCreateTest.cs | 152 +
.../Cache/CacheForkedTest.cs | 1 +
.../Cache/DataRegionMetricsTest.cs | 6 +
.../Cache/DataStorageMetricsTest.cs | 6 +-
.../Cache/PersistenceTest.cs | 166 +-
.../Cache/PersistentStoreTestObsolete.cs | 2 +-
.../Cache/Query/CacheDmlQueriesTest.cs | 2 +-
.../Query/CacheQueriesCodeConfigurationTest.cs | 2 +-
.../Cache/Query/CacheQueriesTest.cs | 6 +-
.../Cache/Query/Linq/CacheLinqTest.Base.cs | 4 +
.../Cache/Query/Linq/CacheLinqTest.Custom.cs | 195 +
.../Cache/Query/Linq/CacheLinqTest.Functions.cs | 48 +
.../Cache/Query/Linq/CacheLinqTest.Misc.cs | 4 +-
.../Cache/Query/Linq/CacheLinqTest.Numerics.cs | 4 +-
.../Client/Cache/CacheClientAsyncWrapper.cs | 1 -
.../Client/Cache/CacheTest.cs | 3 +-
.../Client/Cache/CacheTestNoMeta.cs | 2 +-
.../Client/Cache/CacheTestSsl.cs | 65 +
.../Cache/ClientCacheConfigurationTest.cs | 2 +-
.../Client/Cache/CreateCacheTest.cs | 2 +-
.../Client/Cache/SqlQueryTest.cs | 3 +-
.../Client/ClientConnectionTest.cs | 67 +-
.../Client/ClientTestBase.cs | 4 +-
.../Client/IgniteClientConfigurationTest.cs | 9 +
.../Client/RawSecureSocketTest.cs | 159 +
.../Compute/CancellationTest.cs | 2 +-
.../Compute/ComputeApiTest.cs | 10 +-
.../Config/Apache.Ignite.exe.config.test | 4 +-
.../Config/Apache.Ignite.exe.config.test2 | 2 +-
.../Config/Client/IgniteClientConfiguration.xml | Bin 2482 -> 3144 bytes
.../Config/Client/server-with-ssl.xml | 67 +
.../Config/Client/server.jks | Bin 0 -> 4256 bytes
.../Config/Client/thin-client-cert.pfx | Bin 0 -> 4549 bytes
.../Config/Client/trust.jks | Bin 0 -> 1089 bytes
.../Config/KeyStore/server.jks | Bin 0 -> 3230 bytes
.../Config/KeyStore/trust.jks | Bin 0 -> 2432 bytes
.../Config/full-config.xml | 5 +-
.../Apache.Ignite.Core.Tests/Config/ssl.xml | 65 +
.../Config/start-test-grid1.xml | 55 -
.../Config/start-test-grid2.xml | 46 -
.../Config/start-test-grid3.xml | 44 -
.../DataStructures/AtomicLongTest.cs | 2 +-
.../DataStructures/AtomicReferenceTest.cs | 2 +-
.../DataStructures/AtomicSequenceTest.cs | 2 +-
.../Examples/ExamplesTest.cs | 5 +-
.../Apache.Ignite.Core.Tests/ExecutableTest.cs | 49 +-
.../IgniteConfigurationSerializerTest.cs | 11 +-
.../IgniteConfigurationTest.cs | 6 +-
.../IgniteStartStopTest.cs | 79 +-
.../Apache.Ignite.Core.Tests/IgniteTestBase.cs | 163 -
.../Apache.Ignite.Core.Tests/LoadDllTest.cs | 88 +-
.../Properties/AssemblyInfo.cs | 8 +-
.../Services/ServiceProxyTest.cs | 15 +-
.../Services/ServicesAsyncWrapper.cs | 18 +-
.../Services/ServicesTest.cs | 148 +-
.../Apache.Ignite.Core.Tests/SpringTestBase.cs | 164 +
.../Ssl/SslConfigurationTest.cs | 257 ++
.../dotnet/Apache.Ignite.Core.Tests/TestBase.cs | 61 +
.../Apache.Ignite.Core.Tests/TestRunner.cs | 89 +-
.../TestUtils.Common.cs | 40 +
.../TestUtils.Windows.cs | 3 +-
.../WindowsServiceTest.cs | 58 +-
.../Apache.Ignite.Core.DotNetCore.csproj | 15 +-
.../Apache.Ignite.Core.Schema.nuspec | 2 +-
.../Apache.Ignite.Core.csproj | 69 +-
.../Apache.Ignite.Core.nuspec | 2 +-
.../Cache/Configuration/CacheConfiguration.cs | 2 +
.../Cache/Configuration/QueryEntity.cs | 2 +-
.../Cache/Configuration/QueryField.cs | 2 +-
.../Configuration/QuerySqlFieldAttribute.cs | 2 +-
.../Apache.Ignite.Core/Cache/ICacheMetrics.cs | 9 +
.../Client/Cache/CacheClientConfiguration.cs | 2 +-
.../Client/ISslStreamFactory.cs | 38 +
.../Client/IgniteClientConfiguration.cs | 8 +
.../Client/SslStreamFactory.cs | 110 +
.../Apache.Ignite.Core/Cluster/IBaselineNode.cs | 37 +
.../Apache.Ignite.Core/Cluster/ICluster.cs | 68 +-
.../Apache.Ignite.Core/Cluster/IClusterNode.cs | 3 +-
.../Apache.Ignite.Core/Compute/IComputeTask.cs | 2 +
.../ClientConnectorConfiguration.cs | 47 +-
.../Configuration/DataStorageConfiguration.cs | 3 +-
.../Apache.Ignite.Core/Configuration/WalMode.cs | 13 +-
.../Apache.Ignite.Core/IDataRegionMetrics.cs | 45 +
.../dotnet/Apache.Ignite.Core/IIgnite.cs | 10 +
.../IgniteClientConfigurationSection.xsd | 34 +
.../Apache.Ignite.Core/IgniteConfiguration.cs | 20 +-
.../IgniteConfigurationSection.xsd | 69 +-
.../dotnet/Apache.Ignite.Core/Ignition.cs | 2 +-
.../Impl/Binary/BinaryReader.cs | 8 +-
.../Impl/Binary/BinarySystemHandlers.cs | 16 +
.../Impl/Binary/BinaryTypeId.cs | 7 +-
.../Impl/Binary/Io/BinaryHeapStream.cs | 6 +
.../Impl/Binary/Io/IBinaryStream.cs | 7 +-
.../Impl/Binary/Metadata/BinaryType.cs | 10 +-
.../Impl/Binary/OptimizedMarshallerObject.cs | 54 +
.../Impl/Cache/CacheMetricsImpl.cs | 7 +
.../Query/Continuous/ContinuousQueryFilter.cs | 2 +
.../Cache/ClientCacheConfigurationSerializer.cs | 2 +-
.../Impl/Client/ClientSocket.cs | 70 +-
.../Impl/Cluster/BaselineNode.cs | 89 +
.../Impl/Cluster/ClusterNodeImpl.cs | 38 +-
.../Impl/Common/DelegateTypeDescriptor.cs | 1 +
.../Impl/Common/IgniteArgumentCheck.cs | 1 +
.../Impl/Common/IgniteHome.cs | 29 +-
.../Impl/DataRegionMetrics.cs | 36 +
.../Impl/Handle/HandleRegistry.cs | 1 +
.../dotnet/Apache.Ignite.Core/Impl/Ignite.cs | 76 +-
.../Impl/Memory/PlatformMemoryStream.cs | 12 +-
.../Impl/Services/DynamicServiceProxy.cs | 108 +
.../Impl/Services/ServiceProxyInvoker.cs | 14 +-
.../Impl/Services/ServiceProxySerializer.cs | 37 +-
.../Impl/Services/Services.cs | 43 +-
.../Impl/Ssl/SslFactorySerializer.cs | 65 +
.../Impl/Unmanaged/Jni/Env.cs | 5 +-
.../Impl/Unmanaged/Jni/Jvm.cs | 94 +-
.../Impl/Unmanaged/Jni/JvmDll.cs | 33 +-
.../Properties/AssemblyInfo.cs | 8 +-
.../Apache.Ignite.Core/Services/IServices.cs | 26 +
.../Ssl/ISslContextFactory.cs | 32 +
.../Apache.Ignite.Core/Ssl/SslContextFactory.cs | 170 +
.../Apache.Ignite.EntityFramework.Tests.csproj | 22 +-
.../Properties/AssemblyInfo.cs | 6 +-
.../Apache.Ignite.EntityFramework.nuspec | 2 +-
.../Properties/AssemblyInfo.cs | 8 +-
.../Apache.Ignite.Linq.csproj | 4 +
.../Apache.Ignite.Linq.nuspec | 2 +-
.../Apache.Ignite.Linq/CacheLinqExtensions.cs | 28 +
.../Apache.Ignite.Linq/IUpdateDescriptor.cs | 51 +
.../Apache.Ignite.Linq/Impl/AliasDictionary.cs | 8 +-
.../Impl/CacheFieldsQueryProvider.cs | 23 +-
.../Impl/CacheQueryExpressionVisitor.cs | 43 +-
.../Impl/CacheQueryModelVisitor.cs | 147 +-
.../Apache.Ignite.Linq/Impl/CacheQueryParser.cs | 3 +
.../Impl/Dml/MemberUpdateContainer.cs | 38 +
.../Impl/Dml/UpdateAllExpressionNode.cs | 138 +
.../Impl/Dml/UpdateAllResultOperator.cs | 75 +
.../Apache.Ignite.Linq/Impl/ExpressionWalker.cs | 1 +
.../Properties/AssemblyInfo.cs | 8 +-
.../Apache.Ignite.Log4Net.nuspec | 2 +-
.../Properties/AssemblyInfo.cs | 8 +-
.../Apache.Ignite.NLog.nuspec | 2 +-
.../Properties/AssemblyInfo.cs | 8 +-
modules/platforms/dotnet/Apache.Ignite.sln | 6 +-
.../dotnet/Apache.Ignite/IgniteRunner.cs | 2 +-
.../Apache.Ignite/Properties/AssemblyInfo.cs | 8 +-
.../Apache.Ignite/Service/IgniteService.cs | 13 +-
modules/platforms/dotnet/DEVNOTES.txt | 42 +-
modules/platforms/dotnet/README.md | 6 +-
modules/platforms/dotnet/build-mono.sh | 34 +
modules/platforms/dotnet/build.ps1 | 8 +-
modules/platforms/dotnet/build.sh | 7 +-
.../dotnet/docfx/Apache.Ignite.docfx.json | 2 +-
.../Apache.Ignite.Examples.csproj | 2 +
.../Datagrid/QueryExample.cs | 2 +-
.../Properties/AssemblyInfo.cs | 2 +-
.../Apache.Ignite.Examples/Sql/LinqExample.cs | 5 +-
.../Apache.Ignite.Examples/Sql/SqlDdlExample.cs | 112 +
.../Apache.Ignite.Examples/Sql/SqlDmlExample.cs | 5 +-
.../Apache.Ignite.Examples/Sql/SqlExample.cs | 5 +-
.../ThinClient/ThinClientSqlExample.cs | 222 +
.../Properties/AssemblyInfo.cs | 2 +-
modules/rest-http/pom.xml | 2 +-
.../http/jetty/GridJettyObjectMapper.java | 61 +-
.../http/jetty/GridJettyRestHandler.java | 243 +-
modules/rocketmq/pom.xml | 2 +-
modules/scalar-2.10/pom.xml | 4 +-
modules/scalar/pom.xml | 6 +-
modules/schedule/pom.xml | 2 +-
modules/slf4j/pom.xml | 24 +-
.../apache/ignite/logger/slf4j/Slf4jLogger.java | 44 +-
.../logger/slf4j/Slf4jLoggerMarkerTest.java | 108 +
.../ignite/testsuites/IgniteSlf4jTestSuite.java | 37 +
.../slf4j/src/test/resources/log4j2-test.xml | 38 +
modules/spark-2.10/pom.xml | 155 +-
modules/spark/pom.xml | 187 +-
.../ignite/spark/IgniteDataFrameSettings.scala | 100 +
.../spark/impl/IgniteRelationProvider.scala | 175 +-
.../ignite/spark/impl/IgniteSQLRelation.scala | 93 +-
.../apache/ignite/spark/impl/QueryHelper.scala | 186 +
.../apache/ignite/spark/impl/QueryUtils.scala | 225 ++
.../org/apache/ignite/spark/impl/package.scala | 2 +-
.../sql/ignite/IgniteExternalCatalog.scala | 61 +-
.../src/test/resources/cities_non_unique.json | 6 +
.../ignite/spark/AbstractDataFrameSpec.scala | 43 +-
.../apache/ignite/spark/IgniteCatalogSpec.scala | 14 +-
.../ignite/spark/IgniteDataFrameSuite.scala | 2 +
.../spark/IgniteDataFrameWrongConfigSpec.scala | 4 +-
.../org/apache/ignite/spark/IgniteRDDSpec.scala | 4 +-
...niteSQLDataFrameIgniteSessionWriteSpec.scala | 106 +
.../spark/IgniteSQLDataFrameWriteSpec.scala | 371 ++
modules/spring-data/pom.xml | 2 +-
modules/spring/pom.xml | 2 +-
.../ignite/cache/spring/SpringCacheManager.java | 13 +-
modules/sqlline/pom.xml | 6 +-
modules/ssh/pom.xml | 2 +-
modules/storm/pom.xml | 6 +-
modules/tools/pom.xml | 30 +-
modules/twitter/pom.xml | 2 +-
modules/urideploy/pom.xml | 2 +-
modules/visor-console-2.10/pom.xml | 2 +-
modules/visor-console/pom.xml | 4 +-
.../ignite/visor/commands/VisorConsole.scala | 20 +-
.../commands/cache/VisorCacheCommand.scala | 33 +-
.../commands/cache/VisorCacheScanCommand.scala | 19 +-
.../visor/commands/common/VisorTextTable.scala | 22 +-
.../config/VisorConfigurationCommand.scala | 8 +
.../visor/commands/node/VisorNodeCommand.scala | 123 +-
.../scala/org/apache/ignite/visor/visor.scala | 2 +-
modules/visor-plugins/pom.xml | 2 +-
modules/web-console/.dockerignore | 4 +
modules/web-console/DEVNOTES.txt | 33 +
modules/web-console/backend/app/mongo.js | 1179 +-----
modules/web-console/backend/app/schemas.js | 1175 ++++++
modules/web-console/backend/app/settings.js | 2 -
modules/web-console/backend/index.js | 4 +-
modules/web-console/backend/package.json | 7 +-
modules/web-console/backend/routes/admin.js | 2 +-
modules/web-console/backend/routes/public.js | 3 +-
modules/web-console/backend/services/mails.js | 74 +-
.../backend/services/notifications.js | 9 +-
modules/web-console/backend/services/users.js | 6 +-
.../docker/standalone/docker-compose.yml | 36 -
modules/web-console/e2e/docker-compose.yml | 41 +
modules/web-console/e2e/testcafe/Dockerfile | 34 +
modules/web-console/e2e/testcafe/envtools.js | 192 +
.../e2e/testcafe/fixtures/admin-panel.js | 64 +
.../web-console/e2e/testcafe/fixtures/auth.js | 186 +
.../e2e/testcafe/fixtures/menu-smoke.js | 53 +
.../fixtures/user-profile/credentials.js | 64 +
.../testcafe/fixtures/user-profile/profile.js | 84 +
modules/web-console/e2e/testcafe/helpers.js | 28 +
modules/web-console/e2e/testcafe/package.json | 47 +
.../e2e/testcafe/page-models/PageSignIn.js | 37 +
modules/web-console/e2e/testcafe/roles.js | 33 +
modules/web-console/e2e/testcafe/testcafe.js | 86 +
modules/web-console/e2e/testenv/Dockerfile | 86 +
modules/web-console/e2e/testenv/entrypoint.sh | 21 +
.../web-console/e2e/testenv/nginx/nginx.conf | 55 +
.../e2e/testenv/nginx/web-console.conf | 62 +
modules/web-console/frontend/app/app.js | 27 +-
.../components/cluster-selector/controller.js | 4 +-
.../app/components/grid-export/component.js | 20 +-
.../app/components/grid-no-data/controller.js | 3 +
.../input-dialog/input-dialog.service.js | 3 +-
.../list-editable-transclude/directive.js | 119 +
.../list-editable-transclude/index.js | 23 +
.../app/components/list-editable/controller.js | 1 +
.../app/components/list-editable/index.js | 4 +-
.../app/components/list-editable/template.pug | 6 +-
.../list-of-registered-users/categories.js | 30 +
.../list-of-registered-users/column-defs.js | 83 +
.../list-of-registered-users/controller.js | 384 ++
.../list-of-registered-users/index.js | 18 +-
.../list-of-registered-users.categories.js | 30 -
.../list-of-registered-users.column-defs.js | 83 -
.../list-of-registered-users.controller.js | 386 --
.../list-of-registered-users.scss | 35 -
.../list-of-registered-users.tpl.pug | 66 -
.../list-of-registered-users/style.scss | 34 +
.../list-of-registered-users/template.tpl.pug | 65 +
.../frontend/app/components/page-admin/index.js | 39 +
.../app/components/page-admin/style.scss | 62 +
.../app/components/page-admin/template.tpl.pug | 28 +
.../app/components/page-configure/index.js | 3 +
.../app/components/page-landing/index.js | 65 +
.../app/components/page-landing/style.scss | 104 +
.../app/components/page-landing/template.pug | 66 +
.../page-password-changed/controller.js | 26 +
.../components/page-password-changed/index.js | 41 +
.../components/page-password-changed/style.scss | 42 +
.../page-password-changed/template.pug | 22 +
.../page-password-reset/controller.js | 53 +
.../app/components/page-password-reset/index.js | 57 +
.../components/page-password-reset/style.scss | 31 +
.../components/page-password-reset/template.pug | 80 +
.../app/components/page-profile/component.js | 24 +
.../app/components/page-profile/controller.js | 77 +
.../app/components/page-profile/index.js | 47 +
.../app/components/page-profile/style.scss | 31 +
.../app/components/page-profile/template.pug | 156 +
.../app/components/page-queries/controller.js | 14 +-
.../components/page-queries/template.tpl.pug | 2 +-
.../app/components/page-signin/controller.js | 30 +
.../app/components/page-signin/index.js | 56 +
.../app/components/page-signin/style.scss | 38 +
.../app/components/page-signin/template.pug | 161 +
.../app/components/ui-grid-filters/directive.js | 39 +-
.../app/components/ui-grid-filters/style.scss | 12 +
.../app/components/ui-grid-filters/template.pug | 13 +-
.../components/user-notifications/controller.js | 9 +-
.../components/user-notifications/service.js | 12 +-
.../components/user-notifications/style.scss | 20 +-
.../user-notifications/template.tpl.pug | 5 +
.../components/web-console-header/style.scss | 1 +
.../components/web-console-header/template.pug | 4 +-
.../frontend/app/controllers/auth.controller.js | 29 -
.../frontend/app/data/countries.json | 74 +-
.../app/directives/on-focus-out.directive.js | 8 +-
.../ui-ace-java/ui-ace-java.controller.js | 2 +-
.../ui-ace-spring/ui-ace-spring.controller.js | 2 +-
.../frontend/app/errors/CancellationError.js | 28 +
.../frontend/app/helpers/jade/mixins.pug | 1 +
.../app/modules/agent/AgentManager.service.js | 3 +
.../app/modules/branding/branding.service.js | 2 +-
.../generator/AbstractTransformer.js | 8 +-
.../generator/ConfigurationGenerator.js | 73 +-
.../generator/defaults/Cluster.service.js | 30 +-
.../frontend/app/modules/states/admin.state.js | 15 +-
.../app/modules/states/configuration.state.js | 1 +
.../states/configuration/caches/query.pug | 2 +-
.../configuration/clusters/checkpoint/s3.pug | 28 +-
.../configuration/clusters/client-connector.pug | 33 +-
.../configuration/clusters/communication.pug | 2 +-
.../configuration/clusters/data-storage.pug | 9 +-
.../clusters/general/discovery/s3.pug | 10 +-
.../app/modules/states/password.state.js | 50 -
.../app/modules/states/profile.state.js | 36 -
.../frontend/app/modules/states/signin.state.js | 53 -
.../frontend/app/modules/user/Auth.service.js | 6 +-
.../app/primitives/form-field/checkbox.pug | 30 +
.../app/primitives/form-field/dropdown.pug | 51 +
.../app/primitives/form-field/email.pug | 37 +
.../app/primitives/form-field/error.pug | 29 +
.../app/primitives/form-field/index.pug | 27 +
.../app/primitives/form-field/index.scss | 149 +-
.../app/primitives/form-field/input.pug | 28 +
.../app/primitives/form-field/label.pug | 30 +
.../app/primitives/form-field/number.pug | 46 +
.../app/primitives/form-field/password.pug | 37 +
.../app/primitives/form-field/phone.pug | 35 +
.../frontend/app/primitives/form-field/text.pug | 35 +
.../app/primitives/form-field/tooltip.pug | 18 +
.../frontend/app/primitives/panel/index.scss | 96 +-
.../frontend/app/primitives/tooltip/index.scss | 17 +
.../frontend/app/primitives/ui-grid/index.scss | 38 +-
.../frontend/app/services/Confirm.service.js | 3 +-
.../app/services/ConfirmBatch.service.js | 3 +-
.../frontend/app/services/Countries.service.js | 4 +-
.../frontend/app/services/DefaultState.js | 36 +
.../frontend/app/services/Messages.service.js | 4 +-
.../frontend/app/services/Version.service.js | 6 +-
.../frontend/app/services/exceptionHandler.js | 27 +
.../web-console/frontend/app/services/index.js | 2 +
modules/web-console/frontend/app/vendor.js | 5 +-
.../frontend/controllers/clusters-controller.js | 14 +-
.../frontend/controllers/profile-controller.js | 94 -
modules/web-console/frontend/package.json | 9 +-
.../frontend/public/images/icons/check.svg | 3 +
.../frontend/public/images/icons/collapse.svg | 3 +
.../public/images/icons/exclamation.svg | 3 +
.../frontend/public/images/icons/expand.svg | 3 +
.../frontend/public/images/icons/index.js | 5 +
.../frontend/public/images/main-screenshot.png | Bin 0 -> 86830 bytes
.../public/images/page-landing-carousel-1.png | Bin 0 -> 84434 bytes
.../public/images/page-landing-carousel-2.png | Bin 0 -> 106670 bytes
.../public/images/page-landing-carousel-3.png | Bin 0 -> 70876 bytes
.../public/stylesheets/_bootstrap-custom.scss | 1 -
.../stylesheets/_bootstrap-variables.scss | 2 +-
.../frontend/public/stylesheets/style.scss | 90 +-
.../web-console/frontend/test/ci/.dockerignore | 4 +
modules/web-console/frontend/test/ci/Dockerfile | 36 +
.../frontend/test/ci/docker-compose.yml | 25 +
.../frontend/test/e2e/exampe.test.js | 42 -
.../frontend/test/karma.conf.babel.js | 12 +-
.../frontend/test/protractor.conf.js | 50 -
.../web-console/frontend/views/reset.tpl.pug | 44 -
.../frontend/views/settings/admin.tpl.pug | 28 -
.../frontend/views/settings/profile.tpl.pug | 83 -
.../web-console/frontend/views/signin.tpl.pug | 165 -
.../views/templates/validation-error.tpl.pug | 2 +-
modules/web-console/pom.xml | 5 +-
modules/web-console/web-agent/pom.xml | 2 +-
.../ignite/console/demo/AgentClusterDemo.java | 89 +-
modules/web/ignite-appserver-test/pom.xml | 2 +-
modules/web/ignite-websphere-test/pom.xml | 2 +-
modules/web/pom.xml | 2 +-
modules/yardstick/DEVNOTES-standalone.txt | 10 -
modules/yardstick/DEVNOTES.txt | 11 -
.../config/benchmark-atomic.properties | 2 +-
...benchmark-jdbc-thin-insert-delete.properties | 87 +
.../benchmark-jdbc-thin-select.properties | 86 +
.../benchmark-jdbc-thin-update.properties | 86 +
.../benchmark-jdbcv2-insert-delete.properties | 87 +
.../config/benchmark-jdbcv2-select.properties | 86 +
.../config/benchmark-jdbcv2-update.properties | 88 +
...enchmark-native-sql-insert-delete.properties | 87 +
.../benchmark-native-sql-select.properties | 86 +
.../benchmark-native-sql-update.properties | 86 +
modules/yardstick/config/ignite-jdbc-config.xml | 4 +-
.../ignite-localhost-persistence-config.xml | 71 +
.../benchmark-jdbc-thin-inmemory.properties | 108 +
.../benchmark-jdbc-thin-persistence.properties | 113 +
modules/yardstick/pom-standalone.xml | 56 -
modules/yardstick/pom.xml | 80 +-
.../yardstick/IgniteBenchmarkArguments.java | 35 +-
.../org/apache/ignite/yardstick/IgniteNode.java | 12 +-
.../yardstick/cache/IntMaxValueEntriesTest.java | 4 +-
.../yardstick/jdbc/AbstractJdbcBenchmark.java | 171 +
.../yardstick/jdbc/AbstractNativeBenchmark.java | 37 +
.../yardstick/jdbc/DisjointRangeGenerator.java | 101 +
.../jdbc/JdbcSqlInsertDeleteBenchmark.java | 66 +
.../jdbc/JdbcSqlQueryRangeBenchmark.java | 78 +
.../yardstick/jdbc/JdbcSqlUpdateBenchmark.java | 81 +
.../apache/ignite/yardstick/jdbc/JdbcUtils.java | 55 +
.../jdbc/NativeSqlInsertDeleteBenchmark.java | 61 +
.../jdbc/NativeSqlQueryRangeBenchmark.java | 77 +
.../jdbc/NativeSqlUpdateRangeBenchmark.java | 83 +
.../apache/ignite/yardstick/ml/DataChanger.java | 65 +
...uzzyCMeansDistributedClustererBenchmark.java | 130 +
...gniteFuzzyCMeansLocalClustererBenchmark.java | 93 +
...niteKMeansDistributedClustererBenchmark.java | 75 +
.../IgniteKMeansLocalClustererBenchmark.java | 50 +
.../yardstick/ml/clustering/package-info.java | 22 +
.../IgniteCholeskyDecompositionBenchmark.java | 69 +
.../IgniteEigenDecompositionBenchmark.java | 69 +
.../IgniteLUDecompositionBenchmark.java | 75 +
...niteSingularValueDecompositionBenchmark.java | 59 +
.../ml/decomposition/package-info.java | 22 +
.../ignite/yardstick/ml/knn/Datasets.java | 453 +++
.../knn/IgniteKNNClassificationBenchmark.java | 73 +
.../ml/knn/IgniteKNNRegressionBenchmark.java | 82 +
.../ignite/yardstick/ml/knn/package-info.java | 22 +
.../math/IgniteAbstractMatrixMulBenchmark.java | 120 +
...niteDenseLocalOffHeapMatrixMulBenchmark.java | 32 +
...gniteDenseLocalOnHeapMatrixMulBenchmark.java | 32 +
...parseBlockDistributedMatrixMulBenchmark.java | 32 +
...iteSparseDistributedMatrixMul2Benchmark.java | 74 +
...niteSparseDistributedMatrixMulBenchmark.java | 49 +
.../IgniteSparseLocalMatrixMulBenchmark.java | 32 +
.../ignite/yardstick/ml/math/package-info.java | 22 +
.../ignite/yardstick/ml/package-info.java | 22 +
...iteOLSMultipleLinearRegressionBenchmark.java | 69 +
.../yardstick/ml/regression/package-info.java | 22 +
.../IgniteColumnDecisionTreeGiniBenchmark.java | 70 +
...niteColumnDecisionTreeVarianceBenchmark.java | 71 +
.../yardstick/ml/trees/SplitDataGenerator.java | 426 ++
.../ignite/yardstick/ml/trees/package-info.java | 22 +
.../upload/AbstractNativeBenchmark.java | 114 +
.../upload/AbstractUploadBenchmark.java | 187 +
.../upload/BatchedInsertBenchmark.java | 65 +
.../ignite/yardstick/upload/CopyBenchmark.java | 125 +
.../yardstick/upload/InsertBenchmark.java | 52 +
.../yardstick/upload/NativePutBenchmark.java | 39 +
.../upload/NativeStreamerBenchmark.java | 66 +
.../upload/UploadBenchmarkArguments.java | 158 +
.../yardstick/upload/model/QueryFactory.java | 181 +
.../ignite/yardstick/upload/model/Values10.java | 76 +
.../org/yardstickframework/package-info.java | 25 +
.../probes/TotalTimeProbe.java | 137 +
.../apache/ignite/yardstick/ml/DataChanger.java | 65 -
...uzzyCMeansDistributedClustererBenchmark.java | 130 -
...gniteFuzzyCMeansLocalClustererBenchmark.java | 93 -
...niteKMeansDistributedClustererBenchmark.java | 75 -
.../IgniteKMeansLocalClustererBenchmark.java | 50 -
.../yardstick/ml/clustering/package-info.java | 22 -
.../IgniteCholeskyDecompositionBenchmark.java | 69 -
.../IgniteEigenDecompositionBenchmark.java | 69 -
.../IgniteLUDecompositionBenchmark.java | 75 -
...niteSingularValueDecompositionBenchmark.java | 59 -
.../ml/decomposition/package-info.java | 22 -
.../ignite/yardstick/ml/knn/Datasets.java | 453 ---
.../knn/IgniteKNNClassificationBenchmark.java | 73 -
.../ml/knn/IgniteKNNRegressionBenchmark.java | 82 -
.../ignite/yardstick/ml/knn/package-info.java | 22 -
.../math/IgniteAbstractMatrixMulBenchmark.java | 120 -
...niteDenseLocalOffHeapMatrixMulBenchmark.java | 32 -
...gniteDenseLocalOnHeapMatrixMulBenchmark.java | 32 -
...parseBlockDistributedMatrixMulBenchmark.java | 32 -
...iteSparseDistributedMatrixMul2Benchmark.java | 74 -
...niteSparseDistributedMatrixMulBenchmark.java | 49 -
.../IgniteSparseLocalMatrixMulBenchmark.java | 32 -
.../ignite/yardstick/ml/math/package-info.java | 22 -
.../ignite/yardstick/ml/package-info.java | 22 -
...iteOLSMultipleLinearRegressionBenchmark.java | 69 -
.../yardstick/ml/regression/package-info.java | 22 -
.../IgniteColumnDecisionTreeGiniBenchmark.java | 70 -
...niteColumnDecisionTreeVarianceBenchmark.java | 71 -
.../yardstick/ml/trees/SplitDataGenerator.java | 426 --
.../ignite/yardstick/ml/trees/package-info.java | 22 -
modules/yarn/pom.xml | 13 +-
modules/zeromq/pom.xml | 2 +-
modules/zookeeper/pom.xml | 2 +-
packaging/rpm/SPECS/apache-ignite.spec | 8 +
parent/pom.xml | 103 +-
pom.xml | 16 +-
scripts/update-versions.sh | 35 +
2037 files changed, 111471 insertions(+), 37337 deletions(-)
----------------------------------------------------------------------
[24/50] [abbrv] ignite git commit: IGNITE-7459 Web Console: Hiding
previous scan results on new scan.
Posted by sb...@apache.org.
IGNITE-7459 Web Console: Hiding previous scan results on new scan.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ae2bf3d6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ae2bf3d6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ae2bf3d6
Branch: refs/heads/ignite-zk
Commit: ae2bf3d6fc527e0f85d246abc65547d79226bf19
Parents: 536e825
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Wed Mar 14 11:33:52 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Mar 14 11:33:52 2018 +0700
----------------------------------------------------------------------
.../frontend/app/components/page-queries/controller.js | 3 +--
.../frontend/app/components/page-queries/template.tpl.pug | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae2bf3d6/modules/web-console/frontend/app/components/page-queries/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/controller.js b/modules/web-console/frontend/app/components/page-queries/controller.js
index 2814966..94d62b1 100644
--- a/modules/web-console/frontend/app/components/page-queries/controller.js
+++ b/modules/web-console/frontend/app/components/page-queries/controller.js
@@ -1570,10 +1570,9 @@ export default class {
const filter = paragraph.filter;
const pageSize = paragraph.pageSize;
- paragraph.localQueryMode = local;
-
$scope.scanAvailable(paragraph) && _chooseNode(cacheName, local)
.then((nid) => {
+ paragraph.localQueryMode = local;
paragraph.scanningInProgress = true;
Notebook.save($scope.notebook)
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae2bf3d6/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
index 4419d5c..05bd09b 100644
--- a/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
@@ -277,7 +277,7 @@ mixin paragraph-scan
i.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.checkScanInProgress(true)')
span.tipLabelExecute Scan on selected node
- .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
+ .col-sm-12.sql-result(ng-if='paragraph.queryExecuted() && !paragraph.scanningInProgress' ng-switch='paragraph.resultType()')
.error(ng-switch-when='error') Error: {{paragraph.error.message}}
.empty(ng-switch-when='empty') Result set is empty. Duration: #[b {{paragraph.duration | duration}}]
.table(ng-switch-when='table')
[02/50] [abbrv] ignite git commit: IGNITE-6113 Return back old
implementation instead of 'removeIf' - Fixes #3609.
Posted by sb...@apache.org.
IGNITE-6113 Return back old implementation instead of 'removeIf' - Fixes #3609.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8c3731e8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8c3731e8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8c3731e8
Branch: refs/heads/ignite-zk
Commit: 8c3731e83296e1750c7c098b283c3fd8584b1be4
Parents: 64d920b
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Tue Mar 6 21:22:40 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Mar 6 21:22:40 2018 +0300
----------------------------------------------------------------------
.../cache/distributed/dht/GridDhtLocalPartition.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8c3731e8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 119ea91..7a47f31 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -1194,7 +1194,12 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
void onCacheStopped(int cacheId) {
assert grp.sharedGroup() : grp.cacheOrGroupName();
- rmvQueue.removeIf(e -> e.cacheId() == cacheId);
+ for (Iterator<RemovedEntryHolder> it = rmvQueue.iterator(); it.hasNext();) {
+ RemovedEntryHolder e = it.next();
+
+ if (e.cacheId() == cacheId)
+ it.remove();
+ }
cacheMaps.remove(cacheId);
}
[45/50] [abbrv] ignite git commit: IGNITE-7932: Add SVM example
Posted by sb...@apache.org.
IGNITE-7932: Add SVM example
this closes #3627
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f4ac81a4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f4ac81a4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f4ac81a4
Branch: refs/heads/ignite-zk
Commit: f4ac81a4881d6d4232d6749e0744d987dd2dd3d4
Parents: 094f93b
Author: zaleslaw <za...@gmail.com>
Authored: Fri Mar 16 16:34:42 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Fri Mar 16 16:34:42 2018 +0300
----------------------------------------------------------------------
.../binary/SVMBinaryClassificationExample.java | 233 +++++++++++++++++++
.../examples/ml/svm/binary/package-info.java | 22 ++
.../ignite/examples/ml/svm/package-info.java | 22 ++
3 files changed, 277 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4ac81a4/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
new file mode 100644
index 0000000..52299f9
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.svm.binary;
+
+import java.util.Arrays;
+import java.util.UUID;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
+import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationTrainer;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * Run SVM binary-class classification model over distributed dataset.
+ *
+ * @see SVMLinearBinaryClassificationModel
+ */
+public class SVMBinaryClassificationExample {
+ /** Run example. */
+ public static void main(String[] args) throws InterruptedException {
+ System.out.println();
+ System.out.println(">>> SVM Binary classification model over cached dataset usage example started.");
+ // Start ignite grid.
+ try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+ System.out.println(">>> Ignite grid started.");
+
+ IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+ SVMBinaryClassificationExample.class.getSimpleName(), () -> {
+ IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
+
+ SVMLinearBinaryClassificationTrainer<Integer, double[]> trainer = new SVMLinearBinaryClassificationTrainer<>();
+
+ SVMLinearBinaryClassificationModel mdl = trainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, dataCache),
+ (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+ (k, v) -> v[0],
+ 4);
+
+ System.out.println(">>> SVM model " + mdl);
+
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
+
+ int amountOfErrors = 0;
+ int totalAmount = 0;
+
+ // Build confusion matrix. See https://en.wikipedia.org/wiki/Confusion_matrix
+ int[][] confusionMtx = {{0, 0}, {0, 0}};
+
+
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
+
+ double prediction = mdl.apply(new DenseLocalOnHeapVector(inputs));
+
+ totalAmount++;
+ if(groundTruth != prediction)
+ amountOfErrors++;
+
+
+ int idx1 = (int)prediction == -1.0 ? 0 : 1;
+ int idx2 = (int)groundTruth == -1.0 ? 0 : 1;
+ confusionMtx[idx1][idx2]++;
+
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
+ }
+
+ System.out.println(">>> ---------------------------------");
+
+ System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
+ System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
+ }
+
+ System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtx));
+ });
+
+ igniteThread.start();
+ igniteThread.join();
+ }
+ }
+
+ /**
+ * Fills cache with data and returns it.
+ *
+ * @param ignite Ignite instance.
+ * @return Filled Ignite Cache.
+ */
+ private static IgniteCache<Integer, double[]> getTestCache(Ignite ignite) {
+ CacheConfiguration<Integer, double[]> cacheConfiguration = new CacheConfiguration<>();
+ cacheConfiguration.setName("TEST_" + UUID.randomUUID());
+ cacheConfiguration.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+ IgniteCache<Integer, double[]> cache = ignite.createCache(cacheConfiguration);
+
+ for (int i = 0; i < data.length; i++)
+ cache.put(i, data[i]);
+
+ return cache;
+ }
+
+
+ /** The Iris dataset. */
+ private static final double[][] data = {
+ {-1, 5.1, 3.5, 1.4, 0.2},
+ {-1, 4.9, 3, 1.4, 0.2},
+ {-1, 4.7, 3.2, 1.3, 0.2},
+ {-1, 4.6, 3.1, 1.5, 0.2},
+ {-1, 5, 3.6, 1.4, 0.2},
+ {-1, 5.4, 3.9, 1.7, 0.4},
+ {-1, 4.6, 3.4, 1.4, 0.3},
+ {-1, 5, 3.4, 1.5, 0.2},
+ {-1, 4.4, 2.9, 1.4, 0.2},
+ {-1, 4.9, 3.1, 1.5, 0.1},
+ {-1, 5.4, 3.7, 1.5, 0.2},
+ {-1, 4.8, 3.4, 1.6, 0.2},
+ {-1, 4.8, 3, 1.4, 0.1},
+ {-1, 4.3, 3, 1.1, 0.1},
+ {-1, 5.8, 4, 1.2, 0.2},
+ {-1, 5.7, 4.4, 1.5, 0.4},
+ {-1, 5.4, 3.9, 1.3, 0.4},
+ {-1, 5.1, 3.5, 1.4, 0.3},
+ {-1, 5.7, 3.8, 1.7, 0.3},
+ {-1, 5.1, 3.8, 1.5, 0.3},
+ {-1, 5.4, 3.4, 1.7, 0.2},
+ {-1, 5.1, 3.7, 1.5, 0.4},
+ {-1, 4.6, 3.6, 1, 0.2},
+ {-1, 5.1, 3.3, 1.7, 0.5},
+ {-1, 4.8, 3.4, 1.9, 0.2},
+ {-1, 5, 3, 1.6, 0.2},
+ {-1, 5, 3.4, 1.6, 0.4},
+ {-1, 5.2, 3.5, 1.5, 0.2},
+ {-1, 5.2, 3.4, 1.4, 0.2},
+ {-1, 4.7, 3.2, 1.6, 0.2},
+ {-1, 4.8, 3.1, 1.6, 0.2},
+ {-1, 5.4, 3.4, 1.5, 0.4},
+ {-1, 5.2, 4.1, 1.5, 0.1},
+ {-1, 5.5, 4.2, 1.4, 0.2},
+ {-1, 4.9, 3.1, 1.5, 0.1},
+ {-1, 5, 3.2, 1.2, 0.2},
+ {-1, 5.5, 3.5, 1.3, 0.2},
+ {-1, 4.9, 3.1, 1.5, 0.1},
+ {-1, 4.4, 3, 1.3, 0.2},
+ {-1, 5.1, 3.4, 1.5, 0.2},
+ {-1, 5, 3.5, 1.3, 0.3},
+ {-1, 4.5, 2.3, 1.3, 0.3},
+ {-1, 4.4, 3.2, 1.3, 0.2},
+ {-1, 5, 3.5, 1.6, 0.6},
+ {-1, 5.1, 3.8, 1.9, 0.4},
+ {-1, 4.8, 3, 1.4, 0.3},
+ {-1, 5.1, 3.8, 1.6, 0.2},
+ {-1, 4.6, 3.2, 1.4, 0.2},
+ {-1, 5.3, 3.7, 1.5, 0.2},
+ {-1, 5, 3.3, 1.4, 0.2},
+ {1, 7, 3.2, 4.7, 1.4},
+ {1, 6.4, 3.2, 4.5, 1.5},
+ {1, 6.9, 3.1, 4.9, 1.5},
+ {1, 5.5, 2.3, 4, 1.3},
+ {1, 6.5, 2.8, 4.6, 1.5},
+ {1, 5.7, 2.8, 4.5, 1.3},
+ {1, 6.3, 3.3, 4.7, 1.6},
+ {1, 4.9, 2.4, 3.3, 1},
+ {1, 6.6, 2.9, 4.6, 1.3},
+ {1, 5.2, 2.7, 3.9, 1.4},
+ {1, 5, 2, 3.5, 1},
+ {1, 5.9, 3, 4.2, 1.5},
+ {1, 6, 2.2, 4, 1},
+ {1, 6.1, 2.9, 4.7, 1.4},
+ {1, 5.6, 2.9, 3.6, 1.3},
+ {1, 6.7, 3.1, 4.4, 1.4},
+ {1, 5.6, 3, 4.5, 1.5},
+ {1, 5.8, 2.7, 4.1, 1},
+ {1, 6.2, 2.2, 4.5, 1.5},
+ {1, 5.6, 2.5, 3.9, 1.1},
+ {1, 5.9, 3.2, 4.8, 1.8},
+ {1, 6.1, 2.8, 4, 1.3},
+ {1, 6.3, 2.5, 4.9, 1.5},
+ {1, 6.1, 2.8, 4.7, 1.2},
+ {1, 6.4, 2.9, 4.3, 1.3},
+ {1, 6.6, 3, 4.4, 1.4},
+ {1, 6.8, 2.8, 4.8, 1.4},
+ {1, 6.7, 3, 5, 1.7},
+ {1, 6, 2.9, 4.5, 1.5},
+ {1, 5.7, 2.6, 3.5, 1},
+ {1, 5.5, 2.4, 3.8, 1.1},
+ {1, 5.5, 2.4, 3.7, 1},
+ {1, 5.8, 2.7, 3.9, 1.2},
+ {1, 6, 2.7, 5.1, 1.6},
+ {1, 5.4, 3, 4.5, 1.5},
+ {1, 6, 3.4, 4.5, 1.6},
+ {1, 6.7, 3.1, 4.7, 1.5},
+ {1, 6.3, 2.3, 4.4, 1.3},
+ {1, 5.6, 3, 4.1, 1.3},
+ {1, 5.5, 2.5, 4, 1.3},
+ {1, 5.5, 2.6, 4.4, 1.2},
+ {1, 6.1, 3, 4.6, 1.4},
+ {1, 5.8, 2.6, 4, 1.2},
+ {1, 5, 2.3, 3.3, 1},
+ {1, 5.6, 2.7, 4.2, 1.3},
+ {1, 5.7, 3, 4.2, 1.2},
+ {1, 5.7, 2.9, 4.2, 1.3},
+ {1, 6.2, 2.9, 4.3, 1.3},
+ {1, 5.1, 2.5, 3, 1.1},
+ {1, 5.7, 2.8, 4.1, 1.3},
+ };
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4ac81a4/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/package-info.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/package-info.java
new file mode 100644
index 0000000..22c9ad7
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * SVM Binary Classification Examples.
+ */
+package org.apache.ignite.examples.ml.svm.binary;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4ac81a4/examples/src/main/java/org/apache/ignite/examples/ml/svm/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/package-info.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/package-info.java
new file mode 100644
index 0000000..a2fef75
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * SVM examples.
+ */
+package org.apache.ignite.examples.ml.svm;
\ No newline at end of file
[06/50] [abbrv] ignite git commit: IGNITE-7851: .NET:
CacheConfiguration(string, Type[]) depricated.
Posted by sb...@apache.org.
IGNITE-7851: .NET: CacheConfiguration(string, Type[]) depricated.
This closes #3595
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/caba7cca
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/caba7cca
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/caba7cca
Branch: refs/heads/ignite-zk
Commit: caba7cca8603dc0f7ae7c0faba714f089448bf72
Parents: ed76af2
Author: apopov <ta...@gmail.com>
Authored: Wed Mar 7 12:58:32 2018 +0300
Committer: Igor Sapego <is...@gridgain.com>
Committed: Wed Mar 7 12:58:32 2018 +0300
----------------------------------------------------------------------
.../Cache/Query/CacheQueriesCodeConfigurationTest.cs | 2 +-
.../Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs | 6 ++++--
.../Cache/Query/Linq/CacheLinqTest.Misc.cs | 4 ++--
.../Cache/Query/Linq/CacheLinqTest.Numerics.cs | 4 ++--
.../Apache.Ignite.Core.Tests/Client/Cache/CacheTestNoMeta.cs | 2 +-
.../Cache/Configuration/CacheConfiguration.cs | 2 ++
.../examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs | 2 +-
.../dotnet/examples/Apache.Ignite.Examples/Sql/LinqExample.cs | 5 +++--
.../dotnet/examples/Apache.Ignite.Examples/Sql/SqlExample.cs | 5 +++--
9 files changed, 19 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
index 9768fca..a05923d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
@@ -161,7 +161,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
using (var ignite = Ignition.Start(cfg))
{
var cache = ignite.GetOrCreateCache<int, AttributeQueryPerson>(new CacheConfiguration(CacheName,
- typeof (AttributeQueryPerson)));
+ new QueryEntity(typeof(int), typeof(AttributeQueryPerson))));
Assert.IsNotNull(cache);
http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
index 278d1fd..ceeeb37 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
@@ -616,8 +616,10 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
[Test]
public void TestCustomSchema()
{
- var doubles = GetIgnite().GetOrCreateCache<int, double>(new CacheConfiguration("doubles", typeof(double)));
- var strings = GetIgnite().GetOrCreateCache<int, string>(new CacheConfiguration("strings", typeof(string)));
+ var doubles = GetIgnite().GetOrCreateCache<int, double>(new CacheConfiguration("doubles",
+ new QueryEntity(typeof(int), typeof(double))));
+ var strings = GetIgnite().GetOrCreateCache<int, string>(new CacheConfiguration("strings",
+ new QueryEntity(typeof(int), typeof(string))));
doubles[1] = 36.6;
strings[1] = "foo";
http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Misc.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Misc.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Misc.cs
index 3585281..d9c57d0 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Misc.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Misc.cs
@@ -237,8 +237,8 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
public void TestLocalQuery()
{
// Create partitioned cache
- var cache =
- Ignition.GetIgnite().GetOrCreateCache<int, int>(new CacheConfiguration("partCache", typeof(int))
+ var cache = Ignition.GetIgnite().GetOrCreateCache<int, int>(new CacheConfiguration("partCache",
+ new QueryEntity(typeof(int), typeof(int)))
{
SqlEscapeAll = GetSqlEscapeAll()
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Numerics.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Numerics.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Numerics.cs
index 3f973aa..8dc7ddf 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Numerics.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Numerics.cs
@@ -43,8 +43,8 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
[Test]
public void TestNumerics()
{
- var cache = Ignition.GetIgnite()
- .GetOrCreateCache<int, Numerics>(new CacheConfiguration("numerics", typeof(Numerics))
+ var cache = Ignition.GetIgnite().GetOrCreateCache<int, Numerics>(new CacheConfiguration("numerics",
+ new QueryEntity(typeof(int), typeof(Numerics)))
{
SqlEscapeAll = GetSqlEscapeAll()
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/CacheTestNoMeta.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/CacheTestNoMeta.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/CacheTestNoMeta.cs
index a2ca65d..0a8b146 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/CacheTestNoMeta.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/CacheTestNoMeta.cs
@@ -52,7 +52,7 @@ namespace Apache.Ignite.Core.Tests.Client.Cache
using (var client = Ignition.StartClient(cfg))
{
var serverCache = Ignition.GetIgnite().GetOrCreateCache<int?, Person>(
- new CacheConfiguration("person", typeof(Person)));
+ new CacheConfiguration("person", new QueryEntity(typeof(int?), typeof(Person))));
var clientCache = client.GetCache<int?, Person>(serverCache.Name);
http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
index 4880eb3..06a7d72 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs
@@ -218,12 +218,14 @@ namespace Apache.Ignite.Core.Cache.Configuration
/// <summary>
/// Initializes a new instance of the <see cref="CacheConfiguration"/> class
/// and populates <see cref="QueryEntities"/> according to provided query types.
+ /// This constructor is depricated, please use <see cref="CacheConfiguration(string, QueryEntity[])"/>
/// </summary>
/// <param name="name">Cache name.</param>
/// <param name="queryTypes">
/// Collection of types to be registered as query entities. These types should use
/// <see cref="QuerySqlFieldAttribute"/> to configure query fields and properties.
/// </param>
+ [Obsolete("This constructor is deprecated, please use CacheConfiguration(string, QueryEntity[]) instead.")]
public CacheConfiguration(string name, params Type[] queryTypes) : this(name)
{
QueryEntities = queryTypes.Select(type => new QueryEntity {ValueType = type}).ToArray();
http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs
index 82bd72a..a059b5a 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs
@@ -53,7 +53,7 @@ namespace Apache.Ignite.Examples.Datagrid
Console.WriteLine(">>> Cache query example started.");
var employeeCache = ignite.GetOrCreateCache<int, Employee>(
- new CacheConfiguration(EmployeeCacheName, typeof(Employee)));
+ new CacheConfiguration(EmployeeCacheName, new QueryEntity(typeof(int), typeof(Employee))));
// Populate cache with sample data entries.
PopulateCache(employeeCache);
http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/LinqExample.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/LinqExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/LinqExample.cs
index 9962b20..ee2fd90 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/LinqExample.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/LinqExample.cs
@@ -60,10 +60,11 @@ namespace Apache.Ignite.Examples.Sql
Console.WriteLine(">>> Cache LINQ example started.");
var employeeCache = ignite.GetOrCreateCache<int, Employee>(
- new CacheConfiguration(EmployeeCacheName, typeof(Employee)));
+ new CacheConfiguration(EmployeeCacheName, new QueryEntity(typeof(int), typeof(Employee))));
var employeeCacheColocated = ignite.GetOrCreateCache<AffinityKey, Employee>(
- new CacheConfiguration(EmployeeCacheNameColocated, typeof(Employee)));
+ new CacheConfiguration(EmployeeCacheNameColocated,
+ new QueryEntity(typeof(AffinityKey), typeof(Employee))));
var organizationCache = ignite.GetOrCreateCache<int, Organization>(
new CacheConfiguration(OrganizationCacheName, new QueryEntity(typeof(int), typeof(Organization))));
http://git-wip-us.apache.org/repos/asf/ignite/blob/caba7cca/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlExample.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlExample.cs
index 7222cfb..15b9d40 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlExample.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlExample.cs
@@ -58,10 +58,11 @@ namespace Apache.Ignite.Examples.Sql
Console.WriteLine(">>> Cache query example started.");
var employeeCache = ignite.GetOrCreateCache<int, Employee>(
- new CacheConfiguration(EmployeeCacheName, typeof(Employee)));
+ new CacheConfiguration(EmployeeCacheName, new QueryEntity(typeof(int), typeof(Employee))));
var employeeCacheColocated = ignite.GetOrCreateCache<AffinityKey, Employee>(
- new CacheConfiguration(EmployeeCacheNameColocated, typeof(Employee)));
+ new CacheConfiguration(EmployeeCacheNameColocated,
+ new QueryEntity(typeof(AffinityKey), typeof(Employee))));
var organizationCache = ignite.GetOrCreateCache<int, Organization>(
new CacheConfiguration(OrganizationCacheName, new QueryEntity(typeof(int), typeof(Organization))));
[10/50] [abbrv] ignite git commit: IGNITE-7869 Dynamic start cache by
stored cache data
Posted by sb...@apache.org.
IGNITE-7869 Dynamic start cache by stored cache data
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4ee181ce
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4ee181ce
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4ee181ce
Branch: refs/heads/ignite-zk
Commit: 4ee181ce42972cb5197d780a79e0a96c26fe77e4
Parents: 90c0af8
Author: Anton Kalashnikov <ka...@yandex.ru>
Authored: Mon Mar 12 12:53:36 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Mar 12 12:53:36 2018 +0300
----------------------------------------------------------------------
.../cache/DynamicCacheDescriptor.java | 17 ++-
.../processors/cache/GridCacheProcessor.java | 139 +++++++++----------
.../processors/cache/StoredCacheData.java | 9 +-
3 files changed, 86 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4ee181ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
index 4771c3e..cad8414 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
@@ -284,7 +284,6 @@ public class DynamicCacheDescriptor {
this.rcvdFromVer = rcvdFromVer;
}
-
/**
* @return Start topology version or {@code null} if cache configured statically.
*/
@@ -346,6 +345,22 @@ public class DynamicCacheDescriptor {
}
}
+ /**
+ * Form a {@link StoredCacheData} with all data to correctly restore cache params when its configuration is read
+ * from page store. Essentially, this method takes from {@link DynamicCacheDescriptor} all that's needed to start
+ * cache correctly, leaving out everything else.
+ */
+ public StoredCacheData toStoredData() {
+ assert schema != null;
+
+ StoredCacheData res = new StoredCacheData(cacheConfiguration());
+
+ res.queryEntities(schema().entities());
+ res.sql(sql());
+
+ return res;
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(DynamicCacheDescriptor.class, this, "cacheName", U.maskName(cacheCfg.getName()));
http://git-wip-us.apache.org/repos/asf/ignite/blob/4ee181ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 4648f25..eec964a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -34,6 +34,7 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
+import java.util.stream.Collectors;
import javax.management.MBeanServer;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
@@ -99,8 +100,8 @@ import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaS
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FsyncModeFileWriteAheadLogManager;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FsyncModeFileWriteAheadLogManager;
import org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager;
import org.apache.ignite.internal.processors.cache.query.GridCacheLocalQueryManager;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
@@ -508,7 +509,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
assertParameter(cc.getTransactionManagerLookupClassName() == null,
"transaction manager can not be used with ATOMIC cache");
- if ((cc.getEvictionPolicyFactory() != null || cc.getEvictionPolicy() != null)&& !cc.isOnheapCacheEnabled())
+ if ((cc.getEvictionPolicyFactory() != null || cc.getEvictionPolicy() != null) && !cc.isOnheapCacheEnabled())
throw new IgniteCheckedException("Onheap cache must be enabled if eviction policy is configured [cacheName="
+ U.maskName(cc.getName()) + "]");
@@ -1167,12 +1168,12 @@ public class GridCacheProcessor extends GridProcessorAdapter {
if (log.isInfoEnabled()) {
log.info("Started cache [name=" + cfg.getName() +
- ", id="+cacheCtx.cacheId() +
+ ", id=" + cacheCtx.cacheId() +
(cfg.getGroupName() != null ? ", group=" + cfg.getGroupName() : "") +
", memoryPolicyName=" + memPlcName +
", mode=" + cfg.getCacheMode() +
", atomicity=" + cfg.getAtomicityMode() +
- ", backups=" + cfg.getBackups() +']');
+ ", backups=" + cfg.getBackups() + ']');
}
}
@@ -1189,9 +1190,10 @@ public class GridCacheProcessor extends GridProcessorAdapter {
if (!cache.isNear() && ctx.shared().wal() != null) {
try {
ctx.shared().wal().fsync(null);
- } catch (IgniteCheckedException e) {
+ }
+ catch (IgniteCheckedException e) {
U.error(log, "Failed to flush write-ahead log on cache stop " +
- "[cache=" + ctx.name() + "]", e);
+ "[cache=" + ctx.name() + "]", e);
}
}
@@ -1343,8 +1345,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
* @param cacheObjCtx Cache object context.
* @param affNode {@code True} if local node affinity node.
* @param updatesAllowed Updates allowed flag.
- * @param disabledAfterStart If true, then we will discard restarting state from proxies. If false then we will change
- * state of proxies to restarting
+ * @param disabledAfterStart If true, then we will discard restarting state from proxies. If false then we will
+ * change state of proxies to restarting
* @return Cache context.
* @throws IgniteCheckedException If failed to create cache.
*/
@@ -1845,10 +1847,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
ccfg.setNearConfiguration(reqNearCfg);
}
- StoredCacheData cacheData = toStoredData(desc);
-
if (sharedCtx.pageStore() != null && affNode)
- sharedCtx.pageStore().initializeForCache(desc.groupDescriptor(), cacheData);
+ sharedCtx.pageStore().initializeForCache(desc.groupDescriptor(), desc.toStoredData());
String grpName = startCfg.getGroupName();
@@ -1923,7 +1923,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
if (proxy == null)
continue;
- GridCacheContext<?, ?> cacheCtx = sharedCtx.cacheContext(CU.cacheId(proxy.getName()));
+ GridCacheContext<?, ?> cacheCtx = sharedCtx.cacheContext(CU.cacheId(proxy.getName()));
if (cacheCtx == null)
continue;
@@ -2091,7 +2091,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
for (GridCacheAdapter<?, ?> cache : caches.values()) {
GridCacheContext<?, ?> cacheCtx = cache.context();
- if (cacheCtx.startTopologyVersion().equals(startTopVer) ) {
+ if (cacheCtx.startTopologyVersion().equals(startTopVer)) {
if (!jCacheProxies.containsKey(cacheCtx.name())) {
IgniteCacheProxyImpl newProxy = new IgniteCacheProxyImpl(cache.context(), cache, false);
@@ -2444,8 +2444,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
* @param msg Message.
* @param topVer Current topology version.
* @param curState Current cluster state.
- * @throws IgniteCheckedException If configuration validation failed.
* @return Exchange actions.
+ * @throws IgniteCheckedException If configuration validation failed.
*/
public ExchangeActions onStateChangeRequest(
ChangeGlobalStateMessage msg,
@@ -2595,8 +2595,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
return new GridFinishedFuture<>();
CacheConfiguration ccfg = F.isEmpty(templateName)
- ? getOrCreateConfigFromTemplate(cacheName)
- : getOrCreateConfigFromTemplate(templateName);
+ ? getOrCreateConfigFromTemplate(cacheName)
+ : getOrCreateConfigFromTemplate(templateName);
ccfg.setName(cacheName);
@@ -2785,7 +2785,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
sql,
failIfExists,
failIfNotStarted,
- false);
+ false,
+ null);
if (req != null) {
if (req.clientStartOnly())
@@ -2837,26 +2838,28 @@ public class GridCacheProcessor extends GridProcessorAdapter {
*/
public IgniteInternalFuture<?> dynamicStartCaches(Collection<CacheConfiguration> ccfgList, boolean failIfExists,
boolean checkThreadTx, boolean disabledAfterStart) {
- return dynamicStartCaches(ccfgList, null, failIfExists, checkThreadTx, disabledAfterStart);
+ return dynamicStartCachesByStoredConf(
+ ccfgList.stream().map(StoredCacheData::new).collect(Collectors.toList()),
+ failIfExists,
+ checkThreadTx,
+ disabledAfterStart
+ );
}
/**
* Dynamically starts multiple caches.
*
- * @param ccfgList Collection of cache configuration.
- * @param cacheType Cache type.
+ * @param storedCacheDataList Collection of stored cache data.
* @param failIfExists Fail if exists flag.
* @param checkThreadTx If {@code true} checks that current thread does not have active transactions.
* @param disabledAfterStart If true, cache proxies will be only activated after {@link #restartProxies()}.
* @return Future that will be completed when all caches are deployed.
*/
- private IgniteInternalFuture<?> dynamicStartCaches(
- Collection<CacheConfiguration> ccfgList,
- CacheType cacheType,
+ public IgniteInternalFuture<?> dynamicStartCachesByStoredConf(
+ Collection<StoredCacheData> storedCacheDataList,
boolean failIfExists,
boolean checkThreadTx,
- boolean disabledAfterStart
- ) {
+ boolean disabledAfterStart) {
if (checkThreadTx)
checkEmptyTransactions();
@@ -2864,40 +2867,28 @@ public class GridCacheProcessor extends GridProcessorAdapter {
Map<String, DynamicCacheChangeRequest> clientReqs = null;
try {
- for (CacheConfiguration ccfg : ccfgList) {
- CacheType ct = cacheType;
-
- if (ct == null) {
- if (CU.isUtilityCache(ccfg.getName()))
- ct = CacheType.UTILITY;
- else if (internalCaches.contains(ccfg.getName()))
- ct = CacheType.INTERNAL;
- else if (DataStructuresProcessor.isDataStructureCache(ccfg.getName()))
- ct = CacheType.DATA_STRUCTURES;
- else
- ct = CacheType.USER;
- }
-
+ for (StoredCacheData ccfg : storedCacheDataList) {
DynamicCacheChangeRequest req = prepareCacheChangeRequest(
- ccfg,
- ccfg.getName(),
+ ccfg.config(),
+ ccfg.config().getName(),
null,
- ct,
- false,
+ resolveCacheType(ccfg.config()),
+ ccfg.sql(),
failIfExists,
true,
- disabledAfterStart);
+ disabledAfterStart,
+ ccfg.queryEntities());
if (req != null) {
if (req.clientStartOnly()) {
if (clientReqs == null)
- clientReqs = U.newLinkedHashMap(ccfgList.size());
+ clientReqs = U.newLinkedHashMap(storedCacheDataList.size());
clientReqs.put(req.cacheName(), req);
}
else {
if (srvReqs == null)
- srvReqs = new ArrayList<>(ccfgList.size());
+ srvReqs = new ArrayList<>(storedCacheDataList.size());
srvReqs.add(req);
}
@@ -2931,6 +2922,18 @@ public class GridCacheProcessor extends GridProcessorAdapter {
return new GridFinishedFuture<>();
}
+ /** Resolve cache type for input cacheType */
+ @NotNull private CacheType resolveCacheType(CacheConfiguration ccfg) {
+ if (CU.isUtilityCache(ccfg.getName()))
+ return CacheType.UTILITY;
+ else if (internalCaches.contains(ccfg.getName()))
+ return CacheType.INTERNAL;
+ else if (DataStructuresProcessor.isDataStructureCache(ccfg.getName()))
+ return CacheType.DATA_STRUCTURES;
+ else
+ return CacheType.USER;
+ }
+
/**
* @param cacheName Cache name to destroy.
* @param sql If the cache needs to be destroyed only if it was created as the result of SQL {@code CREATE TABLE}
@@ -2989,6 +2992,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
/**
* Prepares cache stop request.
+ *
* @param cacheName Cache names to destroy.
* @param restart Restart flag.
* @param destroy Cache data destroy flag. Setting to {@code true} will cause removing all cache data from store.
@@ -3006,6 +3010,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
/**
* Starts cache stop request as cache change batch.
+ *
* @param reqs cache stop requests.
* @return compound future.
*/
@@ -3108,7 +3113,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
* @param cacheName Cache name.
* @return Cache type.
*/
- public CacheType cacheType(String cacheName ) {
+ public CacheType cacheType(String cacheName) {
if (CU.isUtilityCache(cacheName))
return CacheType.UTILITY;
else if (internalCaches.contains(cacheName))
@@ -3128,30 +3133,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
assert desc != null;
if (sharedCtx.pageStore() != null && !sharedCtx.kernalContext().clientNode() &&
- CU.isPersistentCache(desc.cacheConfiguration(), sharedCtx.gridConfig().getDataStorageConfiguration())) {
- StoredCacheData data = toStoredData(desc);
-
- sharedCtx.pageStore().storeCacheData(data, true);
- }
- }
-
- /**
- * Form a {@link StoredCacheData} with all data to correctly restore cache params when its configuration
- * is read from page store. Essentially, this method takes from {@link DynamicCacheDescriptor} all that's
- * needed to start cache correctly, leaving out everything else.
- *
- * @param desc Cache descriptor to process.
- * @return {@link StoredCacheData} based on {@code desc}.
- */
- private static StoredCacheData toStoredData(DynamicCacheDescriptor desc) {
- A.notNull(desc, "desc");
-
- StoredCacheData res = new StoredCacheData(desc.cacheConfiguration());
-
- res.queryEntities(desc.schema() == null ? Collections.<QueryEntity>emptyList() : desc.schema().entities());
- res.sql(desc.sql());
-
- return res;
+ CU.isPersistentCache(desc.cacheConfiguration(), sharedCtx.gridConfig().getDataStorageConfiguration()))
+ sharedCtx.pageStore().storeCacheData(desc.toStoredData(), true);
}
/**
@@ -3355,7 +3338,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
/**
* Reset restarting caches.
*/
- public void resetRestartingCaches(){
+ public void resetRestartingCaches() {
cachesInfo.restartingCaches().clear();
}
@@ -3677,7 +3660,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
cache = jCacheProxies.get(cacheName);
}
- return cache != null ? (IgniteCacheProxy<K, V>) cache.gatewayWrapper() : null;
+ return cache != null ? (IgniteCacheProxy<K, V>)cache.gatewayWrapper() : null;
}
/**
@@ -3955,7 +3938,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
@SuppressWarnings("unchecked")
private void registerMbean(Object obj, @Nullable String cacheName, boolean near)
throws IgniteCheckedException {
- if(U.IGNITE_MBEANS_DISABLED)
+ if (U.IGNITE_MBEANS_DISABLED)
return;
assert obj != null;
@@ -3993,7 +3976,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
* @param near Near flag.
*/
private void unregisterMbean(Object o, @Nullable String cacheName, boolean near) {
- if(U.IGNITE_MBEANS_DISABLED)
+ if (U.IGNITE_MBEANS_DISABLED)
return;
assert o != null;
@@ -4141,6 +4124,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
* @param failIfExists Fail if exists flag.
* @param failIfNotStarted If {@code true} fails if cache is not started.
* @param disabledAfterStart If true, cache proxies will be only activated after {@link #restartProxies()}.
+ * @param qryEntities Query entities.
* @return Request or {@code null} if cache already exists.
* @throws IgniteCheckedException if some of pre-checks failed
* @throws CacheExistsException if cache exists and failIfExists flag is {@code true}
@@ -4153,7 +4137,8 @@ public class GridCacheProcessor extends GridProcessorAdapter {
boolean sql,
boolean failIfExists,
boolean failIfNotStarted,
- boolean disabledAfterStart
+ boolean disabledAfterStart,
+ @Nullable Collection<QueryEntity> qryEntities
) throws IgniteCheckedException {
DynamicCacheDescriptor desc = cacheDescriptor(cacheName);
@@ -4208,7 +4193,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
initialize(cfg, cacheObjCtx);
req.startCacheConfiguration(cfg);
- req.schema(new QuerySchema(cfg.getQueryEntities()));
+ req.schema(new QuerySchema(qryEntities != null ? qryEntities : cfg.getQueryEntities()));
}
}
else {
http://git-wip-us.apache.org/repos/asf/ignite/blob/4ee181ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java
index 5a88036..26b4f9b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java
@@ -39,7 +39,7 @@ public class StoredCacheData implements Serializable {
/** Cache configuration. */
@GridToStringInclude
- private final CacheConfiguration<?, ?> ccfg;
+ private CacheConfiguration<?, ?> ccfg;
/** Query entities. */
@GridToStringInclude
@@ -61,6 +61,13 @@ public class StoredCacheData implements Serializable {
}
/**
+ * @param ccfg Cache configuration.
+ */
+ public void config(CacheConfiguration<?, ?> ccfg) {
+ this.ccfg = ccfg;
+ }
+
+ /**
* @return Cache configuration.
*/
public CacheConfiguration<?, ?> config() {
[05/50] [abbrv] ignite git commit: IGNITE-5298 .NET: DML update via
LINQ
Posted by sb...@apache.org.
IGNITE-5298 .NET: DML update via LINQ
This closes #3599
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ed76af24
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ed76af24
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ed76af24
Branch: refs/heads/ignite-zk
Commit: ed76af245e7947b0e701907bde76849971a6a75c
Parents: 3108700
Author: Sergey Stronchinskiy <gu...@gmail.com>
Authored: Wed Mar 7 11:44:51 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Wed Mar 7 11:44:51 2018 +0300
----------------------------------------------------------------------
.../Cache/Query/Linq/CacheLinqTest.Base.cs | 4 +
.../Cache/Query/Linq/CacheLinqTest.Custom.cs | 195 +++++++++++++++++++
.../Apache.Ignite.Linq.csproj | 4 +
.../Apache.Ignite.Linq/CacheLinqExtensions.cs | 28 +++
.../Apache.Ignite.Linq/IUpdateDescriptor.cs | 51 +++++
.../Apache.Ignite.Linq/Impl/AliasDictionary.cs | 8 +-
.../Impl/CacheQueryExpressionVisitor.cs | 43 +++-
.../Impl/CacheQueryModelVisitor.cs | 143 ++++++++++----
.../Apache.Ignite.Linq/Impl/CacheQueryParser.cs | 3 +
.../Impl/Dml/MemberUpdateContainer.cs | 38 ++++
.../Impl/Dml/UpdateAllExpressionNode.cs | 138 +++++++++++++
.../Impl/Dml/UpdateAllResultOperator.cs | 75 +++++++
12 files changed, 680 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Base.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Base.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Base.cs
index d70a7a4..5b56abd 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Base.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Base.cs
@@ -346,6 +346,8 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
[QuerySqlField] public int AliasTest { get; set; }
+ [QuerySqlField] public bool Bool { get; set; }
+
public void WriteBinary(IBinaryWriter writer)
{
writer.WriteInt("age1", Age);
@@ -354,6 +356,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
writer.WriteObject("Address", Address);
writer.WriteTimestamp("Birthday", Birthday);
writer.WriteInt("AliasTest", AliasTest);
+ writer.WriteBoolean("Bool", Bool);
}
public void ReadBinary(IBinaryReader reader)
@@ -364,6 +367,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
Address = reader.ReadObject<Address>("Address");
Birthday = reader.ReadTimestamp("Birthday");
AliasTest = reader.ReadInt("AliasTest");
+ Bool = reader.ReadBoolean("Bool");
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Custom.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Custom.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Custom.cs
index f797b4c..c854f1f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Custom.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Custom.cs
@@ -28,6 +28,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
{
using System;
using System.Linq;
+ using Apache.Ignite.Core.Cache;
using Apache.Ignite.Core.Cache.Configuration;
using Apache.Ignite.Core.Common;
using Apache.Ignite.Linq;
@@ -103,5 +104,199 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
var ex = Assert.Throws<IgniteException>(() => qry.RemoveAll());
Assert.AreEqual("Failed to parse query", ex.Message.Substring(0, 21));
}
+
+ /// <summary>
+ /// Tests the UpdateAll extension without condition.
+ /// </summary>
+ [Test]
+ public void TestUpdateAllUnconditional()
+ {
+ // Use new cache to avoid touching static data.
+ var personCount = 10;
+ var orgCount = 3;
+ var personQueryable = GetPersonCacheQueryable("updateAllTest_Unconditional_Persons", personCount, orgCount);
+ var orgQueryable = GetOrgCacheQueryable("updateAllTest_Unconditional_Org", orgCount);
+
+ var allOrg = orgQueryable.ToArray();
+
+ // Constant value
+ var updated = personQueryable
+ .UpdateAll(d => d.Set(p => p.AliasTest, 7));
+ Assert.AreEqual(personCount, updated);
+ AssertAll(personQueryable, p => p.Value.AliasTest == 7);
+
+ // Expression value - from self
+ updated = personQueryable
+ .UpdateAll(d => d.Set(p => p.AliasTest, e => e.Key));
+ Assert.AreEqual(personCount, updated);
+ AssertAll(personQueryable, p => p.Value.AliasTest == p.Key);
+
+ // Multiple sets
+ var aliasValue = 3;
+ updated = personQueryable
+ .UpdateAll(d => d.Set(p => p.AliasTest, aliasValue).Set(p => p.Name, aliasValue.ToString()));
+ Assert.AreEqual(personCount, updated);
+ AssertAll(personQueryable, p => p.Value.AliasTest == aliasValue && p.Value.Name == aliasValue.ToString());
+
+ // Expression value - subquery with same cache
+ updated = personQueryable
+ .UpdateAll(d => d.Set(p => p.AliasTest,
+ e => personQueryable.Where(ie => ie.Key == e.Key).Select(ie => ie.Key).First()));
+ Assert.AreEqual(personCount, updated);
+ AssertAll(personQueryable, p => p.Value.AliasTest == p.Key);
+
+ // Expression value - subquery with other cache
+ updated = personQueryable
+ .UpdateAll(d => d.Set(p => p.AliasTest, p => orgQueryable.Count(o => o.Value.Id > p.Key)));
+ Assert.AreEqual(personCount, updated);
+ AssertAll(personQueryable, p => p.Value.AliasTest == allOrg.Count(o => o.Value.Id > p.Key));
+
+ updated = personQueryable
+ .UpdateAll(d => d.Set(p => p.Name,
+ e => orgQueryable.Where(o => o.Key == e.Value.OrganizationId).Select(o => o.Value.Name).First()));
+ Assert.AreEqual(personCount, updated);
+ AssertAll(personQueryable,
+ p => p.Value.Name == allOrg.Where(o => o.Key == p.Value.OrganizationId).Select(o => o.Value.Name)
+ .FirstOrDefault());
+
+ // Expression value - Contains subquery with other cache
+ updated = personQueryable
+ .UpdateAll(d => d.Set(p => p.Bool,
+ p => orgQueryable.Select(o => o.Key).Contains(p.Value.OrganizationId)));
+ Assert.AreEqual(personCount, updated);
+ AssertAll(personQueryable,
+ p => p.Value.Bool == allOrg.Select(o => o.Key).Contains(p.Value.OrganizationId));
+
+ // Row number limit.
+ var name = "rowLimit" + 2;
+ updated = personQueryable
+ .Take(2)
+ .UpdateAll(d => d.Set(p => p.Name, name));
+ Assert.AreEqual(2, updated);
+ Assert.AreEqual(2, personQueryable.Count(p => p.Value.Name == name));
+ }
+
+ /// <summary>
+ /// Tests the UpdateAll extension with condition.
+ /// </summary>
+ [Test]
+ public void TestUpdateAllWithCondition()
+ {
+ // ReSharper disable AccessToModifiedClosure
+ // Use new cache to avoid touching static data.
+ var personQueryable = GetPersonCacheQueryable("updateAllTest_WithCondition_Persons", 10);
+
+ // Simple conditional
+ var aliasValue = 777;
+ var updated = personQueryable
+ .Where(p => p.Key > 8)
+ .UpdateAll(d => d.Set(p => p.AliasTest, aliasValue));
+ Assert.AreEqual(2, updated);
+ AssertAll(personQueryable,
+ p => p.Key <= 8 && p.Value.AliasTest != aliasValue || p.Value.AliasTest == aliasValue);
+
+ // Conditional with limit
+ aliasValue = 8888;
+ updated = personQueryable
+ .Where(p => p.Key > 1)
+ .Take(3)
+ .UpdateAll(d => d.Set(p => p.AliasTest, aliasValue));
+ Assert.AreEqual(3, updated);
+ Assert.AreEqual(3, personQueryable.ToArray().Count(p => p.Value.AliasTest == aliasValue));
+ // ReSharper restore AccessToModifiedClosure
+ }
+
+ /// <summary>
+ /// Tests not supported queries for the UpdateAll extension .
+ /// </summary>
+ [Test]
+ public void TestUpdateAllUnsupported()
+ {
+ // Use new cache to avoid touching static data.
+ var personQueryable = GetPersonCacheQueryable("updateAllTest_Unsupported_Persons", 10);
+
+ // Skip is not supported with DELETE.
+ var nex = Assert.Throws<NotSupportedException>(
+ () => personQueryable.Skip(1).UpdateAll(d => d.Set(p => p.Age, 15)));
+ Assert.AreEqual("UpdateAll can not be combined with result operators (other than Take): SkipResultOperator",
+ nex.Message);
+
+ // Multiple result operators are not supported with DELETE.
+ nex = Assert.Throws<NotSupportedException>(() =>
+ personQueryable.Skip(1).Take(1).UpdateAll(d => d.Set(p => p.Age, 15)));
+ Assert.AreEqual(
+ "UpdateAll can not be combined with result operators (other than Take): SkipResultOperator, " +
+ "TakeResultOperator, UpdateAllResultOperator", nex.Message);
+
+ // Joins are not supported in H2.
+ var qry = personQueryable
+ .Where(x => x.Key == 7)
+ .Join(GetPersonCache().AsCacheQueryable(), p => p.Key, p => p.Key, (p1, p2) => p1);
+
+ var ex = Assert.Throws<IgniteException>(() => qry.UpdateAll(d => d.Set(p => p.Age, 15)));
+ Assert.AreEqual("Failed to parse query", ex.Message.Substring(0, 21));
+ }
+
+ /// <summary>
+ /// Gets filled persons cache queryable
+ /// </summary>
+ private IQueryable<ICacheEntry<int, Person>> GetPersonCacheQueryable(string cacheName, int personCount,
+ int? orgCount = null)
+ {
+ var cache = GetCache<Person>(cacheName);
+
+ cache.PutAll(Enumerable.Range(1, personCount).ToDictionary(x => x,
+ x => new Person(x, x.ToString())
+ {
+ Birthday = DateTime.UtcNow.AddDays(personCount - x),
+ OrganizationId = x % orgCount ?? 0
+ }));
+
+ return cache.AsCacheQueryable();
+ }
+
+ /// <summary>
+ /// Asserts that all values in cache correspond to predicate
+ /// </summary>
+ private static void AssertAll(
+ // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Local
+ IQueryable<ICacheEntry<int, Person>> personQueryable,
+ // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Local
+ Func<ICacheEntry<int, Person>, bool> predicate)
+ {
+ Assert.IsTrue(personQueryable.ToArray().All(predicate));
+ }
+
+ /// <summary>
+ /// Gets filled organization cache queryable
+ /// </summary>
+ private IQueryable<ICacheEntry<int, Organization>> GetOrgCacheQueryable(string cacheName, int orgCount)
+ {
+ var cache = GetCache<Organization>(cacheName);
+ var allOrg = Enumerable.Range(1, orgCount)
+ .Select(x => new Organization
+ {
+ Id = x,
+ Name = x.ToString()
+ })
+ .ToList();
+
+ allOrg.ForEach(x => cache.Put(x.Id, x));
+
+ return cache.AsCacheQueryable();
+ }
+
+ /// <summary>
+ /// Gets cache of <see cref="T"/>
+ /// </summary>
+ private ICache<int, T> GetCache<T>(string cacheName)
+ {
+ return Ignition.GetIgnite().GetOrCreateCache<int, T>(new CacheConfiguration(cacheName,
+ new QueryEntity(typeof(int), typeof(T)))
+ {
+ SqlEscapeAll = GetSqlEscapeAll(),
+ CacheMode = CacheMode.Replicated
+ });
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.csproj
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.csproj b/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.csproj
index fc13914..d5a2d83 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Apache.Ignite.Linq.csproj
@@ -64,7 +64,10 @@
<Compile Include="Impl\CacheQueryExpressionVisitor.cs" />
<Compile Include="Impl\CacheQueryModelVisitor.cs" />
<Compile Include="Impl\CacheQueryParser.cs" />
+ <Compile Include="Impl\Dml\MemberUpdateContainer.cs" />
+ <Compile Include="Impl\Dml\UpdateAllExpressionNode.cs" />
<Compile Include="Impl\Dml\RemoveAllExpressionNode.cs" />
+ <Compile Include="Impl\Dml\UpdateAllResultOperator.cs" />
<Compile Include="Impl\Dml\RemoveAllResultOperator.cs" />
<Compile Include="Impl\EnumerableHelper.cs" />
<Compile Include="Impl\ICacheQueryableInternal.cs" />
@@ -73,6 +76,7 @@
<Compile Include="Impl\SqlTypes.cs" />
<Compile Include="Impl\ExpressionWalker.cs" />
<Compile Include="Impl\JoinInnerSequenceParameterNotNullExpressionVisitor.cs" />
+ <Compile Include="IUpdateDescriptor.cs" />
<Compile Include="Package-Info.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="QueryOptions.cs" />
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs
index 935666f..2b65d85 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs
@@ -193,5 +193,33 @@ namespace Apache.Ignite.Linq
return query.Provider.Execute<int>(Expression.Call(null, method, query.Expression,
Expression.Quote(predicate)));
}
+
+ /// <summary>
+ /// Updates all rows that are matched by the specified query.
+ /// <para />
+ /// This method results in "UPDATE" distributed SQL query, performing bulk update
+ /// (as opposed to fetching all rows locally).
+ /// </summary>
+ /// <typeparam name="TKey">Key type.</typeparam>
+ /// <typeparam name="TValue">Value type.</typeparam>
+ /// <param name="query">The query.</param>
+ /// <param name="updateDescription">The update description.</param>
+ /// <returns>Affected row count.</returns>
+ public static int UpdateAll<TKey, TValue>(this IQueryable<ICacheEntry<TKey, TValue>> query,
+ Expression<Func<IUpdateDescriptor<TKey,TValue>, IUpdateDescriptor<TKey,TValue>>> updateDescription)
+ {
+ IgniteArgumentCheck.NotNull(query, "query");
+ IgniteArgumentCheck.NotNull(updateDescription, "updateDescription");
+
+ var method = UpdateAllExpressionNode.UpdateAllMethodInfo
+ .MakeGenericMethod(typeof(TKey), typeof(TValue)); // TODO: cache?
+
+ return query.Provider.Execute<int>(Expression.Call(null, method, new[]
+ {
+ query.Expression,
+ Expression.Quote(updateDescription)
+ }));
+ }
+
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/IUpdateDescriptor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/IUpdateDescriptor.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/IUpdateDescriptor.cs
new file mode 100644
index 0000000..2327b99
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/IUpdateDescriptor.cs
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Linq
+{
+ using System;
+ using Apache.Ignite.Core.Cache;
+
+ /// <summary>
+ /// Dummy interface to provide update description for
+ /// <see
+ /// cref="CacheLinqExtensions.UpdateAll{TKey,TValue}" />
+ /// </summary>
+ /// <typeparam name="TKey">Key type.</typeparam>
+ /// <typeparam name="TValue">Value type.</typeparam>
+ public interface IUpdateDescriptor<out TKey, out TValue>
+ {
+ /// <summary>
+ /// Specifies member update with constant
+ /// </summary>
+ /// <typeparam name="TProp">Member type</typeparam>
+ /// <param name="selector">Member selector</param>
+ /// <param name="value">New value</param>
+ /// <returns></returns>
+ IUpdateDescriptor<TKey, TValue> Set<TProp>(Func<TValue, TProp> selector, TProp value);
+
+ /// <summary>
+ /// Specifies member update with expression
+ /// </summary>
+ /// <typeparam name="TProp">Member type</typeparam>
+ /// <param name="selector">Member selector</param>
+ /// <param name="valueBuilder">New value generator</param>
+ /// <returns></returns>
+ IUpdateDescriptor<TKey, TValue> Set<TProp>(Func<TValue, TProp> selector,
+ Func<ICacheEntry<TKey, TValue>, TProp> valueBuilder);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/AliasDictionary.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/AliasDictionary.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/AliasDictionary.cs
index 8902e7c..15710db 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/AliasDictionary.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/AliasDictionary.cs
@@ -20,6 +20,7 @@ namespace Apache.Ignite.Linq.Impl
using System;
using System.Collections.Generic;
using System.Diagnostics;
+ using System.Linq;
using System.Linq.Expressions;
using System.Text;
using Remotion.Linq.Clauses;
@@ -49,11 +50,14 @@ namespace Apache.Ignite.Linq.Impl
/// <summary>
/// Pushes current aliases to stack.
/// </summary>
- public void Push()
+ /// <param name="copyAliases">Flag indicating that current aliases should be copied</param>
+ public void Push(bool copyAliases)
{
_stack.Push(_tableAliases);
- _tableAliases = new Dictionary<IQuerySource, string>();
+ _tableAliases = copyAliases
+ ? _tableAliases.ToDictionary(p => p.Key, p => p.Value)
+ : new Dictionary<IQuerySource, string>();
}
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs
index 4caefe1..c5c887f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryExpressionVisitor.cs
@@ -54,21 +54,34 @@ namespace Apache.Ignite.Linq.Impl
/** */
private readonly bool _includeAllFields;
+ /** */
+ private readonly bool _visitEntireSubQueryModel;
+
/// <summary>
/// Initializes a new instance of the <see cref="CacheQueryExpressionVisitor" /> class.
/// </summary>
/// <param name="modelVisitor">The _model visitor.</param>
- /// <param name="useStar">Flag indicating that star '*' qualifier should be used
- /// for the whole-table select instead of _key, _val.</param>
- /// <param name="includeAllFields">Flag indicating that star '*' qualifier should be used
- /// for the whole-table select as well as _key, _val.</param>
- public CacheQueryExpressionVisitor(CacheQueryModelVisitor modelVisitor, bool useStar, bool includeAllFields)
+ /// <param name="useStar">
+ /// Flag indicating that star '*' qualifier should be used
+ /// for the whole-table select instead of _key, _val.
+ /// </param>
+ /// <param name="includeAllFields">
+ /// Flag indicating that star '*' qualifier should be used
+ /// for the whole-table select as well as _key, _val.
+ /// </param>
+ /// <param name="visitEntireSubQueryModel">
+ /// Flag indicating that subquery
+ /// should be visited as full query
+ /// </param>
+ public CacheQueryExpressionVisitor(CacheQueryModelVisitor modelVisitor, bool useStar, bool includeAllFields,
+ bool visitEntireSubQueryModel)
{
Debug.Assert(modelVisitor != null);
_modelVisitor = modelVisitor;
_useStar = useStar;
_includeAllFields = includeAllFields;
+ _visitEntireSubQueryModel = visitEntireSubQueryModel;
}
/// <summary>
@@ -548,12 +561,18 @@ namespace Apache.Ignite.Linq.Impl
var subQueryModel = expression.QueryModel;
var contains = subQueryModel.ResultOperators.FirstOrDefault() as ContainsResultOperator;
-
+
// Check if IEnumerable.Contains is used.
- if (subQueryModel.ResultOperators.Count == 1 && contains != null)
+ if (subQueryModel.ResultOperators.Count == 1 && contains != null)
{
VisitContains(subQueryModel, contains);
}
+ else if (_visitEntireSubQueryModel)
+ {
+ ResultBuilder.Append("(");
+ _modelVisitor.VisitQueryModel(subQueryModel, false, true);
+ ResultBuilder.Append(")");
+ }
else
{
// This happens when New expression uses a subquery, in a GroupBy.
@@ -579,7 +598,15 @@ namespace Apache.Ignite.Linq.Impl
Visit(contains.Item);
ResultBuilder.Append(" IN (");
- _modelVisitor.VisitQueryModel(subQueryModel);
+ if (_visitEntireSubQueryModel)
+ {
+ _modelVisitor.VisitQueryModel(subQueryModel, false, true);
+ }
+ else
+ {
+ _modelVisitor.VisitQueryModel(subQueryModel);
+ }
+
ResultBuilder.Append(")");
}
else
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
index 223eea6..da250e9 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryModelVisitor.cs
@@ -103,26 +103,30 @@ namespace Apache.Ignite.Linq.Impl
/// <summary>
/// Visits the query model.
/// </summary>
- private void VisitQueryModel(QueryModel queryModel, bool includeAllFields)
+ internal void VisitQueryModel(QueryModel queryModel, bool includeAllFields, bool copyAliases = false)
{
- _aliases.Push();
+ _aliases.Push(copyAliases);
- var hasDelete = VisitRemoveOperator(queryModel);
-
- if (!hasDelete)
+ var lastResultOp = queryModel.ResultOperators.LastOrDefault();
+ if (lastResultOp is RemoveAllResultOperator)
+ {
+ VisitRemoveOperator(queryModel);
+ }
+ else if(lastResultOp is UpdateAllResultOperator)
+ {
+ VisitUpdateAllOperator(queryModel);
+ }
+ else
{
// SELECT
_builder.Append("select ");
// TOP 1 FLD1, FLD2
VisitSelectors(queryModel, includeAllFields);
- }
- // FROM ... WHERE ... JOIN ...
- base.VisitQueryModel(queryModel);
+ // FROM ... WHERE ... JOIN ...
+ base.VisitQueryModel(queryModel);
- if (!hasDelete)
- {
// UNION ...
ProcessResultOperatorsEnd(queryModel);
}
@@ -131,42 +135,68 @@ namespace Apache.Ignite.Linq.Impl
}
/// <summary>
- /// Visits the remove operator. Returns true if it is present.
+ /// Visits the remove operator.
/// </summary>
- private bool VisitRemoveOperator(QueryModel queryModel)
+ private void VisitRemoveOperator(QueryModel queryModel)
{
var resultOps = queryModel.ResultOperators;
- if (resultOps.LastOrDefault() is RemoveAllResultOperator)
+ _builder.Append("delete ");
+
+ if (resultOps.Count == 2)
{
- _builder.Append("delete ");
+ var resOp = resultOps[0] as TakeResultOperator;
- if (resultOps.Count == 2)
- {
- var resOp = resultOps[0] as TakeResultOperator;
+ if (resOp == null)
+ throw new NotSupportedException(
+ "RemoveAll can not be combined with result operators (other than Take): " +
+ resultOps[0].GetType().Name);
- if (resOp == null)
- {
- throw new NotSupportedException(
- "RemoveAll can not be combined with result operators (other than Take): " +
- resultOps[0].GetType().Name);
- }
+ _builder.Append("top ");
+ BuildSqlExpression(resOp.Count);
+ _builder.Append(" ");
+ }
+ else if (resultOps.Count > 2)
+ {
+ throw new NotSupportedException(
+ "RemoveAll can not be combined with result operators (other than Take): " +
+ string.Join(", ", resultOps.Select(x => x.GetType().Name)));
+ }
- _builder.Append("top ");
- BuildSqlExpression(resOp.Count);
- _builder.Append(" ");
- }
- else if (resultOps.Count > 2)
- {
+ // FROM ... WHERE ... JOIN ...
+ base.VisitQueryModel(queryModel);
+ }
+
+ /// <summary>
+ /// Visits the UpdateAll operator.
+ /// </summary>
+ private void VisitUpdateAllOperator(QueryModel queryModel)
+ {
+ var resultOps = queryModel.ResultOperators;
+
+ _builder.Append("update ");
+
+ // FROM ... WHERE ...
+ base.VisitQueryModel(queryModel);
+
+ if (resultOps.Count == 2)
+ {
+ var resOp = resultOps[0] as TakeResultOperator;
+
+ if (resOp == null)
throw new NotSupportedException(
- "RemoveAll can not be combined with result operators (other than Take): " +
- string.Join(", ", resultOps.Select(x => x.GetType().Name)));
- }
+ "UpdateAll can not be combined with result operators (other than Take): " +
+ resultOps[0].GetType().Name);
- return true;
+ _builder.Append("limit ");
+ BuildSqlExpression(resOp.Count);
+ }
+ else if (resultOps.Count > 2)
+ {
+ throw new NotSupportedException(
+ "UpdateAll can not be combined with result operators (other than Take): " +
+ string.Join(", ", resultOps.Select(x => x.GetType().Name)));
}
-
- return false;
}
/// <summary>
@@ -394,18 +424,28 @@ namespace Apache.Ignite.Linq.Impl
{
base.VisitMainFromClause(fromClause, queryModel);
- _builder.AppendFormat("from ");
+ var isUpdateQuery = queryModel.ResultOperators.LastOrDefault() is UpdateAllResultOperator;
+ if (!isUpdateQuery)
+ {
+ _builder.Append("from ");
+ }
+
ValidateFromClause(fromClause);
_aliases.AppendAsClause(_builder, fromClause).Append(" ");
var i = 0;
foreach (var additionalFrom in queryModel.BodyClauses.OfType<AdditionalFromClause>())
{
- _builder.AppendFormat(", ");
+ _builder.Append(", ");
ValidateFromClause(additionalFrom);
VisitAdditionalFromClause(additionalFrom, queryModel, i++);
}
+
+ if (isUpdateQuery)
+ {
+ BuildSetClauseForUpdateAll(queryModel);
+ }
}
/// <summary>
@@ -535,7 +575,6 @@ namespace Apache.Ignite.Linq.Impl
}
}
-
/// <summary>
/// Visists Join clause in case of join with local collection
/// </summary>
@@ -651,9 +690,33 @@ namespace Apache.Ignite.Linq.Impl
/// <summary>
/// Builds the SQL expression.
/// </summary>
- private void BuildSqlExpression(Expression expression, bool useStar = false, bool includeAllFields = false)
+ private void BuildSqlExpression(Expression expression, bool useStar = false, bool includeAllFields = false,
+ bool visitSubqueryModel = false)
+ {
+ new CacheQueryExpressionVisitor(this, useStar, includeAllFields, visitSubqueryModel).Visit(expression);
+ }
+
+ /// <summary>
+ /// Builds SET clause of UPDATE statement
+ /// </summary>
+ private void BuildSetClauseForUpdateAll(QueryModel queryModel)
{
- new CacheQueryExpressionVisitor(this, useStar, includeAllFields).Visit(expression);
+ var updateAllResultOperator = queryModel.ResultOperators.LastOrDefault() as UpdateAllResultOperator;
+ if (updateAllResultOperator != null)
+ {
+ _builder.Append("set ");
+ var first = true;
+ foreach (var update in updateAllResultOperator.Updates)
+ {
+ if (!first) _builder.Append(", ");
+ first = false;
+ BuildSqlExpression(update.Selector);
+ _builder.Append(" = ");
+ BuildSqlExpression(update.Value, visitSubqueryModel: true);
+ }
+
+ _builder.Append(" ");
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryParser.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryParser.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryParser.cs
index 17ec0a3..3f69cce 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryParser.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheQueryParser.cs
@@ -66,6 +66,9 @@ namespace Apache.Ignite.Linq.Impl
methodInfoRegistry.Register(RemoveAllExpressionNode.GetSupportedMethods(),
typeof(RemoveAllExpressionNode));
+ methodInfoRegistry.Register(UpdateAllExpressionNode.GetSupportedMethods(),
+ typeof(UpdateAllExpressionNode));
+
return new CompoundNodeTypeProvider(new INodeTypeProvider[]
{
methodInfoRegistry,
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/MemberUpdateContainer.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/MemberUpdateContainer.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/MemberUpdateContainer.cs
new file mode 100644
index 0000000..bf8a5fd
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/MemberUpdateContainer.cs
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+namespace Apache.Ignite.Linq.Impl.Dml
+{
+ using System.Linq.Expressions;
+
+ /// <summary>
+ /// Contains information about member update
+ /// </summary>
+ internal struct MemberUpdateContainer
+ {
+ /// <summary>
+ /// Gets or sets member selector
+ /// </summary>
+ public Expression Selector { get; set; }
+
+ /// <summary>
+ /// Gets or sets member new value
+ /// </summary>
+ public Expression Value { get; set; }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllExpressionNode.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllExpressionNode.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllExpressionNode.cs
new file mode 100644
index 0000000..8cfb552
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllExpressionNode.cs
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Linq.Impl.Dml
+{
+ using System;
+ using System.Collections.Generic;
+ using System.Diagnostics.CodeAnalysis;
+ using System.Linq;
+ using System.Linq.Expressions;
+ using System.Reflection;
+ using Remotion.Linq.Clauses;
+ using Remotion.Linq.Clauses.Expressions;
+ using Remotion.Linq.Parsing.ExpressionVisitors;
+ using Remotion.Linq.Parsing.Structure.IntermediateModel;
+
+ /// <summary>
+ /// Represents a <see cref="MethodCallExpression" /> for
+ /// <see cref="CacheLinqExtensions.UpdateAll{TKey,TValue}" />.
+ /// When user calls UpdateAll, this node is generated.
+ /// </summary>
+ internal sealed class UpdateAllExpressionNode : ResultOperatorExpressionNodeBase
+ {
+ /// <summary>
+ /// The UpdateAll method.
+ /// </summary>
+ public static readonly MethodInfo UpdateAllMethodInfo = typeof(CacheLinqExtensions)
+ .GetMethods()
+ .Single(x => x.Name == "UpdateAll");
+
+ //** */
+ private static readonly MethodInfo[] SupportedMethods = {UpdateAllMethodInfo};
+
+ //** */
+ private readonly LambdaExpression _updateDescription;
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="UpdateAllExpressionNode" /> class.
+ /// </summary>
+ /// <param name="parseInfo">The parse information.</param>
+ /// <param name="updateDescription">Expression with update description info</param>
+ public UpdateAllExpressionNode(MethodCallExpressionParseInfo parseInfo,
+ LambdaExpression updateDescription)
+ : base(parseInfo, null, null)
+ {
+ _updateDescription = updateDescription;
+ }
+
+ /** <inheritdoc /> */
+ [ExcludeFromCodeCoverage]
+ public override Expression Resolve(ParameterExpression inputParameter, Expression expressionToBeResolved,
+ ClauseGenerationContext clauseGenerationContext)
+ {
+ throw CreateResolveNotSupportedException();
+ }
+
+ /** <inheritdoc /> */
+ protected override ResultOperatorBase CreateResultOperator(ClauseGenerationContext clauseGenerationContext)
+ {
+ if (_updateDescription.Parameters.Count != 1)
+ throw new NotSupportedException("Expression is not supported for UpdateAll: " +
+ _updateDescription);
+
+ var querySourceRefExpression = (QuerySourceReferenceExpression) Source.Resolve(
+ _updateDescription.Parameters[0],
+ _updateDescription.Parameters[0],
+ clauseGenerationContext);
+
+ var cacheEntryType = querySourceRefExpression.Type;
+ var querySourceAccessValue =
+ Expression.MakeMemberAccess(querySourceRefExpression, cacheEntryType.GetMember("Value").First());
+
+ if (!(_updateDescription.Body is MethodCallExpression))
+ throw new NotSupportedException("Expression is not supported for UpdateAll: " +
+ _updateDescription.Body);
+
+ var updates = new List<MemberUpdateContainer>();
+
+ var methodCall = (MethodCallExpression) _updateDescription.Body;
+ while (methodCall != null)
+ {
+ if (methodCall.Arguments.Count != 2)
+ throw new NotSupportedException("Method is not supported for UpdateAll: " + methodCall);
+
+ var selectorLambda = (LambdaExpression) methodCall.Arguments[0];
+ var selector = ReplacingExpressionVisitor.Replace(selectorLambda.Parameters[0], querySourceAccessValue,
+ selectorLambda.Body);
+
+ var newValue = methodCall.Arguments[1];
+ switch (newValue.NodeType)
+ {
+ case ExpressionType.Constant:
+ break;
+ case ExpressionType.Lambda:
+ var newValueLambda = (LambdaExpression) newValue;
+ newValue = ReplacingExpressionVisitor.Replace(newValueLambda.Parameters[0],
+ querySourceRefExpression, newValueLambda.Body);
+ break;
+ default:
+ throw new NotSupportedException("Value expression is not supported for UpdateAll: "
+ + newValue);
+ }
+
+ updates.Add(new MemberUpdateContainer
+ {
+ Selector = selector,
+ Value = newValue
+ });
+
+ methodCall = methodCall.Object as MethodCallExpression;
+ }
+
+ return new UpdateAllResultOperator(updates);
+ }
+
+ /// <summary>
+ /// Gets the supported methods.
+ /// </summary>
+ public static IEnumerable<MethodInfo> GetSupportedMethods()
+ {
+ return SupportedMethods;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed76af24/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllResultOperator.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllResultOperator.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllResultOperator.cs
new file mode 100644
index 0000000..be682b0
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/Dml/UpdateAllResultOperator.cs
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Linq.Impl.Dml
+{
+ using System;
+ using System.Collections.Generic;
+ using System.Diagnostics.CodeAnalysis;
+ using System.Linq;
+ using System.Linq.Expressions;
+ using Remotion.Linq.Clauses;
+ using Remotion.Linq.Clauses.ResultOperators;
+ using Remotion.Linq.Clauses.StreamedData;
+
+ /// <summary>
+ /// Represents an operator for <see cref="CacheLinqExtensions.UpdateAll{TKey,TValue}"/>.
+ /// </summary>
+ internal sealed class UpdateAllResultOperator : ValueFromSequenceResultOperatorBase
+ {
+ /// <summary>
+ /// Gets the members updates
+ /// </summary>
+ public MemberUpdateContainer[] Updates { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of <see cref="UpdateAllResultOperator" />
+ /// </summary>
+ /// <param name="updates">members updates</param>
+ public UpdateAllResultOperator(IEnumerable<MemberUpdateContainer> updates)
+ {
+ Updates = updates.ToArray();
+ }
+
+ /** <inheritdoc /> */
+ public override IStreamedDataInfo GetOutputDataInfo(IStreamedDataInfo inputInfo)
+ {
+ return new StreamedScalarValueInfo(typeof(int));
+ }
+
+ /** <inheritdoc /> */
+ [ExcludeFromCodeCoverage]
+ public override ResultOperatorBase Clone(CloneContext cloneContext)
+ {
+ return new UpdateAllResultOperator(Updates);
+ }
+
+ /** <inheritdoc /> */
+ [ExcludeFromCodeCoverage]
+ public override void TransformExpressions(Func<Expression, Expression> transformation)
+ {
+ // No-op.
+ }
+
+ /** <inheritdoc /> */
+ [ExcludeFromCodeCoverage]
+ public override StreamedValue ExecuteInMemory<T>(StreamedSequence sequence)
+ {
+ throw new NotSupportedException("UpdateAll is not supported for in-memory sequences.");
+ }
+ }
+}
\ No newline at end of file
[21/50] [abbrv] ignite git commit: IGNITE-7765 Ensure that
walSegmentSize doesn't overflow.
Posted by sb...@apache.org.
IGNITE-7765 Ensure that walSegmentSize doesn't overflow.
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8da7c9e1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8da7c9e1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8da7c9e1
Branch: refs/heads/ignite-zk
Commit: 8da7c9e1b90d14fbce971a6566ce369b795ed30b
Parents: 1f8ec79
Author: Ivan Daschinskiy <iv...@gmail.com>
Authored: Tue Mar 13 17:32:02 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Tue Mar 13 17:32:30 2018 +0300
----------------------------------------------------------------------
.../configuration/DataStorageConfiguration.java | 8 ++--
.../DataStorageConfigurationValidationTest.java | 46 ++++++++++++++++++++
.../ignite/testsuites/IgniteCacheTestSuite.java | 2 +
3 files changed, 53 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8da7c9e1/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
index 4a3dbbf..a433760 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
@@ -528,16 +528,18 @@ public class DataStorageConfiguration implements Serializable {
* @return WAL segment size.
*/
public int getWalSegmentSize() {
- return walSegmentSize <= 0 ? DFLT_WAL_SEGMENT_SIZE : walSegmentSize;
+ return walSegmentSize == 0 ? DFLT_WAL_SEGMENT_SIZE : walSegmentSize;
}
/**
* Sets size of a WAL segment.
*
- * @param walSegmentSize WAL segment size. 64 MB is used by default. Maximum value is 2Gb
- * @return {@code this} for chaining.
+ * @param walSegmentSize WAL segment size. 64 MB is used by default. Maximum value is 2Gb.
+ * @return {@code This} for chaining.
*/
public DataStorageConfiguration setWalSegmentSize(int walSegmentSize) {
+ A.ensure(walSegmentSize >= 0, "WAL segment size must be non-negative and less than 2 Gb.");
+
this.walSegmentSize = walSegmentSize;
return this;
http://git-wip-us.apache.org/repos/asf/ignite/blob/8da7c9e1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java
new file mode 100644
index 0000000..7f667ee
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/DataStorageConfigurationValidationTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.concurrent.Callable;
+import junit.framework.TestCase;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Tests assertions in DataStorageConfiguration.
+ */
+public class DataStorageConfigurationValidationTest extends TestCase {
+ /**
+ * Tests {@link DataStorageConfiguration#walSegmentSize} property assertion.
+ *
+ * @throws Exception If failed.
+ */
+ public void testWalSegmentSizeOveflow() throws Exception {
+ final DataStorageConfiguration cfg = new DataStorageConfiguration();
+
+ GridTestUtils.assertThrows(null, new Callable<Void>() {
+ /** {@inheritDoc} */
+ @Override public Void call() {
+ cfg.setWalSegmentSize(1 << 31);
+
+ return null;
+ }
+ }, IllegalArgumentException.class, null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8da7c9e1/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index b309b0a..bb397f7 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -82,6 +82,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheSwapPreloadSelfTest;
import org.apache.ignite.internal.processors.cache.GridCacheTtlManagerSelfTest;
import org.apache.ignite.internal.processors.cache.GridCacheTxPartitionedLocalStoreSelfTest;
import org.apache.ignite.internal.processors.cache.GridDataStorageConfigurationConsistencySelfTest;
+import org.apache.ignite.internal.processors.cache.DataStorageConfigurationValidationTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicInvokeTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalInvokeTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalWithStoreInvokeTest;
@@ -229,6 +230,7 @@ public class IgniteCacheTestSuite extends TestSuite {
suite.addTestSuite(GridCacheConfigurationValidationSelfTest.class);
suite.addTestSuite(GridCacheConfigurationConsistencySelfTest.class);
suite.addTestSuite(GridDataStorageConfigurationConsistencySelfTest.class);
+ suite.addTestSuite(DataStorageConfigurationValidationTest.class);
suite.addTestSuite(GridCacheJdbcBlobStoreSelfTest.class);
suite.addTestSuite(GridCacheJdbcBlobStoreMultithreadedSelfTest.class);
suite.addTestSuite(JdbcTypesDefaultTransformerTest.class);
[14/50] [abbrv] ignite git commit: IGNITE-7878 .NET: Un-Ignore
TestStartDefault
Posted by sb...@apache.org.
IGNITE-7878 .NET: Un-Ignore TestStartDefault
This reverts commit 18c6b4a5d079874e9c156fa39564970ad3c31914.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ed2435c8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ed2435c8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ed2435c8
Branch: refs/heads/ignite-zk
Commit: ed2435c81e435750fb2640aac1ea2a9ae40bce2d
Parents: 318ffe5
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Mon Mar 12 16:21:41 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Mon Mar 12 16:21:41 2018 +0300
----------------------------------------------------------------------
.../dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed2435c8/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
index 5ef095c..792b33d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
@@ -47,7 +47,6 @@ namespace Apache.Ignite.Core.Tests
///
/// </summary>
[Test]
- [Ignore("IGNITE-7878")]
public void TestStartDefault()
{
var cfg = new IgniteConfiguration {JvmClasspath = TestUtils.CreateTestClasspath()};
[29/50] [abbrv] ignite git commit: IGNITE-7947 Not all OWNING
partitions saved in PartitionAllocationMap during checkpoint - Fixes #3632.
Posted by sb...@apache.org.
IGNITE-7947 Not all OWNING partitions saved in PartitionAllocationMap during checkpoint - Fixes #3632.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/798c01bd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/798c01bd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/798c01bd
Branch: refs/heads/ignite-zk
Commit: 798c01bd6d8427ec85ebbef4c0bec4fdcacfee29
Parents: 48a6623
Author: Eduard Shangareev <es...@gridgain.com>
Authored: Wed Mar 14 16:48:23 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Mar 14 16:48:23 2018 +0300
----------------------------------------------------------------------
.../preloader/GridDhtPartitionExchangeId.java | 3 ++
.../persistence/GridCacheOffheapManager.java | 41 ++++++++++++++------
2 files changed, 33 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/798c01bd/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java
index 33728d3..741386b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java
@@ -212,6 +212,9 @@ public class GridDhtPartitionExchangeId implements Message, Comparable<GridDhtPa
/** {@inheritDoc} */
@Override public boolean equals(Object o) {
+ if (o == null)
+ return false;
+
if (o == this)
return true;
http://git-wip-us.apache.org/repos/asf/ignite/blob/798c01bd/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index eb0c30f..d3b73bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -51,7 +51,6 @@ import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteHistoricalIterator;
import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
@@ -73,7 +72,6 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.query.GridQueryRowCacheCleaner;
import org.apache.ignite.internal.util.lang.GridCursor;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
@@ -162,14 +160,8 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
boolean metaWasUpdated = false;
- for (CacheDataStore store : partDataStores.values()) {
- RowStore rowStore = store.rowStore();
-
- if (rowStore == null)
- continue;
-
+ for (CacheDataStore store : partDataStores.values())
metaWasUpdated |= saveStoreMetadata(store, ctx, !metaWasUpdated, false);
- }
}
/**
@@ -210,8 +202,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
if (beforeDestroy)
state = GridDhtPartitionState.EVICTED;
else {
- part = grp.topology().localPartition(store.partId(),
- AffinityTopologyVersion.NONE, false, true);
+ part = getPartition(store);
if (part != null && part.state() != GridDhtPartitionState.EVICTED)
state = part.state();
@@ -337,12 +328,40 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
pageMem.releasePage(grpId, partMetaId, partMetaPage);
}
}
+ else if (needSnapshot)
+ tryAddEmptyPartitionToSnapshot(store, ctx);;
}
+ else if (needSnapshot)
+ tryAddEmptyPartitionToSnapshot(store, ctx);
return wasSaveToMeta;
}
/**
+ * Check that we need to snapshot this partition and add it to map.
+ *
+ * @param store Store.
+ * @param ctx Snapshot context.
+ */
+ private void tryAddEmptyPartitionToSnapshot(CacheDataStore store, Context ctx) {
+ if (getPartition(store).state() == OWNING) {
+ ctx.partitionStatMap().put(
+ new GroupPartitionId(grp.groupId(), store.partId()),
+ new PagesAllocationRange(0, 0));
+ }
+ }
+
+ /**
+ * @param store Store.
+ *
+ * @return corresponding to store local partition
+ */
+ private GridDhtLocalPartition getPartition(CacheDataStore store) {
+ return grp.topology().localPartition(store.partId(),
+ AffinityTopologyVersion.NONE, false, true);
+ }
+
+ /**
* Loads cache sizes for all caches in shared group.
*
* @param pageMem page memory to perform operations on pages.
[16/50] [abbrv] ignite git commit: .NET: Update DEVNOTES - better
.NET Core test commands
Posted by sb...@apache.org.
.NET: Update DEVNOTES - better .NET Core test commands
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3cb9e258
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3cb9e258
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3cb9e258
Branch: refs/heads/ignite-zk
Commit: 3cb9e2582470dbb4ace357e45d48158a3ccba642
Parents: 447a8a9
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Mon Mar 12 20:12:13 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Mon Mar 12 20:12:13 2018 +0300
----------------------------------------------------------------------
modules/platforms/dotnet/DEVNOTES.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3cb9e258/modules/platforms/dotnet/DEVNOTES.txt
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/DEVNOTES.txt b/modules/platforms/dotnet/DEVNOTES.txt
index 0726c46..199fe90 100644
--- a/modules/platforms/dotnet/DEVNOTES.txt
+++ b/modules/platforms/dotnet/DEVNOTES.txt
@@ -49,9 +49,9 @@ Getting started:
./build.sh
* Run tests:
cd Apache.Ignite.Core.Tests.DotNetCore
- dotnet test
+ dotnet test --logger "console;verbosity=normal"
* Run specific test:
- dotnet test --filter CacheTest
+ dotnet test --filter CacheTest --logger "console;verbosity=normal"
* IDE: Open Apache.Ignite.DotNetCore.sln
[19/50] [abbrv] ignite git commit: IGNITE-7638 Implemented robin-hood
hashing for FullPageIdTable - Fixes #3481.
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
new file mode 100644
index 0000000..fd4e0e6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
@@ -0,0 +1,654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.util.function.BiConsumer;
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.util.GridLongList;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.NotNull;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR;
+import static org.apache.ignite.IgniteSystemProperties.getFloat;
+import static org.apache.ignite.internal.util.GridUnsafe.getInt;
+import static org.apache.ignite.internal.util.GridUnsafe.getLong;
+import static org.apache.ignite.internal.util.GridUnsafe.putInt;
+import static org.apache.ignite.internal.util.GridUnsafe.putLong;
+
+/**
+ * Loaded pages mapping to relative pointer based on Robin Hood hashing: backward shift deletion algorithm. <br>
+ * Performance of initial Robin Hood hashing could be greatly improved with only a little change to the removal
+ * method.<br> Instead of replacing entries with 'Removed' fake entries on deletion, backward shift deletion variant
+ * for the Robin Hood hashing algorithm does shift backward all the entries following the entry to delete until
+ * either an empty bucket, or a bucket with a DIB of 0 (distance to initial bucket).<br>
+ *
+ * Every deletion will shift backwards entries and therefore decrease their respective DIBs by 1
+ * (all their initial DIB values would be >= 1)<br>
+ *
+ * This implementation stores ideal bucket with entry value itself.<br>
+ *
+ */
+public class RobinHoodBackwardShiftHashMap implements LoadedPagesMap {
+ /** Load factor. */
+ private static final float LOAD_FACTOR = getFloat(IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR, 2.5f);
+
+ /** Size of count of entries (value returned by size() method). */
+ private static final int MAPSIZE_SIZE = 4;
+
+ /** Padding to provide read/write from word beginning for each cell. Change this to 0 if padding is not required. */
+ private static final int CELL_PADDING = 4;
+
+ /** Padding to provide read/write from word beginning. Change this to 0 if padding is not required. */
+ private static final int MAPSIZE_PADDING = 4;
+
+ /** Count of entries offset starting from base address. */
+ private static final int MAPSIZE_OFFSET = 0;
+
+ /** Size of initial/ideal bucket (cell to store value to avoid probing other cells followed). */
+ private static final int IDEAL_BUCKET_SIZE = 4;
+
+ /** Offset of initial/ideal bucket starting from entry base. */
+ private static final int IDEAL_BUCKET_OFFSET = 0;
+
+ /** Group ID size. */
+ private static final int GRP_ID_SIZE = 4;
+
+ /** Group ID offset from entry base. */
+ private static final int GRP_ID_OFFSET = IDEAL_BUCKET_OFFSET + IDEAL_BUCKET_SIZE;
+
+ /** Page ID size. */
+ private static final int PAGE_ID_SIZE = 8;
+
+ /** Page ID offset from entry base. */
+ private static final int PAGE_ID_OFFSET = GRP_ID_OFFSET + GRP_ID_SIZE;
+
+ /** Value size. */
+ private static final int VALUE_SIZE = 8;
+
+ /** Value offset from entry base. */
+ private static final int VALUE_OFFSET = PAGE_ID_OFFSET + PAGE_ID_SIZE;
+
+ /** Version (tag/generation) offset from entry base. */
+ private static final int VERSION_SIZE = 4;
+
+ /** Version (tag/generation) offset from entry base. */
+ private static final int VERSION_OFFSET = VALUE_OFFSET + VALUE_SIZE;
+
+ /** Page ID used for empty bucket. */
+ private static final long EMPTY_PAGE_ID = 0;
+
+ /** Cache Group ID used for empty bucket. */
+ private static final int EMPTY_CACHE_GRP_ID = 0;
+
+ /** Bytes required for storing one entry (cell). */
+ private static final int BYTES_PER_CELL = IDEAL_BUCKET_SIZE
+ + GRP_ID_SIZE + PAGE_ID_SIZE
+ + VALUE_SIZE + VERSION_SIZE
+ + CELL_PADDING;
+
+ /** Number of buckets, indicates range of scan memory, max probe count and maximum map size. */
+ private final int numBuckets;
+
+ /** Base address of map content. */
+ private long baseAddr;
+
+ /**
+ * @param elementsCnt Maximum elements can be stored in map, its maximum size.
+ * @return Estimated memory size required for this map to store the given number of elements.
+ */
+ public static long requiredMemory(long elementsCnt) {
+ float loadFactor = LOAD_FACTOR;
+
+ assert loadFactor != 0;
+
+ return requiredMemoryByBuckets((long)(elementsCnt * loadFactor));
+ }
+
+ /**
+ * @param numBuckets Number of buckets (cells) to store, capacity.
+ * @return required size to allocate, based on number of buckets (cells) to store in map, its capacity.
+ */
+ static long requiredMemoryByBuckets(long numBuckets) {
+ return numBuckets * BYTES_PER_CELL + MAPSIZE_SIZE + MAPSIZE_PADDING;
+ }
+
+ /**
+ * Creates map in preallocated unsafe memory segment.
+ *
+ * @param baseAddr Base buffer address.
+ * @param size Size available for map, number of buckets (cells) to store will be determined accordingly.
+ */
+ public RobinHoodBackwardShiftHashMap(long baseAddr, long size) {
+ this.numBuckets = (int)((size - MAPSIZE_SIZE - MAPSIZE_PADDING) / BYTES_PER_CELL);
+ this.baseAddr = baseAddr;
+
+ GridUnsafe.setMemory(baseAddr, size, (byte)0);
+ }
+
+ /**
+ * @param idx cell index.
+ * @return base cell (bucket) address in buffer.
+ */
+ private long entryBase(int idx) {
+ assert idx >= 0 && idx < numBuckets;
+
+ return baseAddr + MAPSIZE_SIZE + MAPSIZE_PADDING + (long)idx * BYTES_PER_CELL;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long get(int grpId, long pageId, int reqVer, long absent, long outdated) {
+ assert grpId != EMPTY_CACHE_GRP_ID;
+
+ // initial index is also ideal for searhed element
+ int idxInit = U.safeAbs(FullPageId.hashCode(grpId, pageId)) % numBuckets;
+
+ for (int i = 0; i < numBuckets; i++) {
+ int idxCurr = (idxInit + i) % numBuckets;
+
+ final long base = entryBase(idxCurr);
+ final int distanceFromInit = distance(idxCurr, idxInit);
+
+ final int curGrpId = getGrpId(base);
+ final long curPageId = getPageId(base);
+
+ final int dibCurEntry = distance(idxCurr, getIdealBucket(base));
+
+ if (isEmpty(curGrpId, curPageId))
+ return absent;
+ else if (curGrpId == grpId && curPageId == pageId) {
+ //equal value found
+ long actualVer = getVersion(base);
+ boolean freshVal = actualVer >= reqVer;
+
+ return freshVal ? getValue(base) : outdated;
+ }
+ else if (dibCurEntry < distanceFromInit) {
+ //current entry has quite good position, it would be swapped at hypothetical insert of current value
+ return absent;
+ }
+ }
+
+ return absent;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void put(int grpId, long pageId, long val, int ver) {
+ assert grpId != 0;
+
+ // initial index is also ideal for inserted element
+ int idxInit = U.safeAbs(FullPageId.hashCode(grpId, pageId)) % numBuckets;
+
+ int swapCnt = 0;
+
+ int grpIdToInsert = grpId;
+ long pageIdToInsert = pageId;
+ long valToInsert = val;
+ int verToInsert = ver;
+ long idxIdealToInsert = idxInit;
+
+ for (int i = 0; i < numBuckets; i++) {
+ int idxCurr = (idxInit + i) % numBuckets;
+
+ final long base = entryBase(idxCurr);
+ final int dibEntryToInsert = distance(idxCurr, idxInit);
+
+ final int curGrpId = getGrpId(base);
+ final long curPageId = getPageId(base);
+ final int curIdealBucket = getIdealBucket(base);
+ final long curVal = getValue(base);
+ final int curVer = getVersion(base);
+ final int dibCurEntry = distance(idxCurr, curIdealBucket);
+
+ if (isEmpty(curGrpId, curPageId)) {
+ setCellValue(base, idxIdealToInsert, grpIdToInsert, pageIdToInsert, valToInsert, verToInsert);
+
+ setSize(size() + 1);
+
+ return;
+ }
+ else if (curGrpId == grpIdToInsert && curPageId == pageIdToInsert) {
+ if (swapCnt != 0)
+ throw new IllegalStateException("Swapped " + swapCnt + " times. Entry: " + dumpEntry(idxCurr));
+
+ setValue(base, valToInsert);
+
+ return; //equal value found
+ }
+ else if (dibCurEntry < dibEntryToInsert) {
+ //swapping *toInsert and state in bucket: save cur state to bucket
+ setCellValue(base, idxIdealToInsert, grpIdToInsert, pageIdToInsert, valToInsert, verToInsert);
+
+ idxIdealToInsert = curIdealBucket;
+ pageIdToInsert = curPageId;
+ grpIdToInsert = curGrpId;
+ valToInsert = curVal;
+ verToInsert = curVer;
+
+ swapCnt++;
+ }
+ }
+
+ // no free space left
+ throw new IgniteOutOfMemoryException("No room for a new key");
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean remove(int grpId, long pageId) {
+ assert grpId != EMPTY_CACHE_GRP_ID;
+
+ int idxInit = U.safeAbs(FullPageId.hashCode(grpId, pageId)) % numBuckets;
+
+ int idxEqualValFound = -1;
+ for (int i = 0; i < numBuckets; i++) {
+ int idxCurr = (idxInit + i) % numBuckets;
+
+ final long base = entryBase(idxCurr);
+ final int dibEntryToInsert = distance(idxCurr, idxInit);
+
+ final int curGrpId = getGrpId(base);
+ final long curPageId = getPageId(base);
+ final int curIdealBucket = getIdealBucket(base);
+ final int dibCurEntry = distance(idxCurr, curIdealBucket);
+
+ if (isEmpty(curGrpId, curPageId))
+ return false;
+ else if (curGrpId == grpId && curPageId == pageId) {
+ idxEqualValFound = idxCurr;
+
+ break; //equal value found
+ }
+ else if (dibCurEntry < dibEntryToInsert) {
+ //If our value was present in map we had already found it.
+ return false;
+ }
+ }
+
+ setSize(size() - 1);
+
+ doBackwardShift(idxEqualValFound);
+
+ return true;
+ }
+
+ /**
+ * Runs backward shifts from current index to .
+ *
+ * @param idxRmv removed index.
+ */
+ private void doBackwardShift(int idxRmv) {
+ assert idxRmv >= 0;
+
+ //scanning rest of map to perform backward shifts
+ for (int i = 0; i < numBuckets - 1; i++) {
+ int idxCurr = (idxRmv + i) % numBuckets;
+ int idxNext = (idxRmv + i + 1) % numBuckets;
+
+ long baseCurr = entryBase(idxCurr);
+
+ long baseNext = entryBase(idxNext);
+ final int nextGrpId = getGrpId(baseNext);
+ final long nextPageId = getPageId(baseNext);
+ final int nextIdealBucket = getIdealBucket(baseNext);
+ final int nextEntryVer = getVersion(baseNext);
+
+ if (isEmpty(nextGrpId, nextPageId)
+ || distance(idxNext, nextIdealBucket) == 0) {
+ setEmpty(baseCurr);
+
+ return;
+ }
+ else
+ setCellValue(baseCurr, nextIdealBucket, nextGrpId, nextPageId, getValue(baseNext), nextEntryVer);
+ }
+
+ int lastShiftedIdx = (idxRmv - 1) % numBuckets;
+
+ setEmpty(entryBase(lastShiftedIdx));
+ }
+
+ /** {@inheritDoc} */
+ @Override public ReplaceCandidate getNearestAt(final int idxStart) {
+ for (int i = 0; i < numBuckets; i++) {
+ int idxCurr = (idxStart + i) % numBuckets;
+
+ if (isEmptyAt(idxCurr))
+ continue;
+
+ long base = entryBase(idxCurr);
+
+ return new ReplaceCandidate(getVersion(base), getValue(base), getFullPageId(base));
+ }
+
+ return null;
+ }
+
+ /**
+ * @param idx Index to test.
+ * @return {@code True} if value is not provided in cell having index.
+ */
+ private boolean isEmptyAt(int idx) {
+ long base = entryBase(idx);
+
+ return isEmpty(getGrpId(base), getPageId(base));
+ }
+
+ /** {@inheritDoc} */
+ @Override public long refresh(int grpId, long pageId, int ver) {
+ assert grpId != EMPTY_CACHE_GRP_ID;
+
+ int idxInit = U.safeAbs(FullPageId.hashCode(grpId, pageId)) % numBuckets;
+
+ for (int i = 0; i < numBuckets; i++) {
+ int idxCurr = (idxInit + i) % numBuckets;
+
+ final long base = entryBase(idxCurr);
+ final int distanceFromInit = distance(idxCurr, idxInit);
+
+ final int curGrpId = getGrpId(base);
+ final long curPageId = getPageId(base);
+ final int curIdealBucket = getIdealBucket(base);
+ final int dibCurEntry = distance(idxCurr, curIdealBucket);
+
+ if (isEmpty(curGrpId, curPageId))
+ break; // break to fail
+ else if (curGrpId == grpId && curPageId == pageId) {
+ //equal value found
+ long actualVer = getVersion(base);
+
+ boolean freshVal = actualVer >= ver;
+
+ if (freshVal) {
+ throw new IllegalArgumentException("Fresh element found at " +
+ dumpEntry(idxCurr) + " during search of cell to refresh. " +
+ "Refresh should be called for existent outdated element. ");
+ }
+
+ setVersion(base, ver);
+
+ return getValue(base);
+ }
+ else if (dibCurEntry < distanceFromInit) {
+ //current entry has quite good position, it would be swapped at hypothetical insert of current value
+
+ break;
+ }
+ }
+
+ throw new IllegalArgumentException("Element not found group ID: " + grpId + ", page ID: " + pageId +
+ " during search of cell to refresh. Refresh should be called for existent outdated element. ");
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridLongList removeIf(int startIdxToClear, int endIdxToClear, KeyPredicate keyPred) {
+ assert endIdxToClear >= startIdxToClear
+ : "Start and end indexes are not consistent: {" + startIdxToClear + ", " + endIdxToClear + "}";
+
+ int sz = endIdxToClear - startIdxToClear;
+
+ GridLongList list = new GridLongList(sz);
+
+ for (int idx = startIdxToClear; idx < endIdxToClear; idx++) {
+ long base = entryBase(idx);
+
+ int grpId = getGrpId(base);
+ long pageId = getPageId(base);
+
+ if (isEmpty(grpId, pageId))
+ continue; // absent value, no removal required
+
+ if (!keyPred.test(grpId, pageId))
+ continue; // not matched value, no removal required
+
+ long valAt = getValue(base);
+
+ setSize(size() - 1);
+
+ doBackwardShift(idx);
+
+ list.add(valAt);
+
+ idx--; //Need recheck current cell because of backward shift
+ }
+
+ return list;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int capacity() {
+ return numBuckets;
+ }
+
+ /**
+ * @param curr current selected index to store value.
+ * @param baseIdx base or ideal bucket to store entry value to avoid probing.
+ * @return distance between cells, or 0 if cell is ideal.
+ */
+ private int distance(int curr, int baseIdx) {
+ int diff = curr - baseIdx;
+
+ if (diff < 0)
+ return diff + numBuckets;
+
+ return diff;
+ }
+
+ /**
+ * @param grpId Cache group ID.
+ * @param pageId Page ID.
+ * @return {@code true} if group & page id indicates cell has state 'Empty'.
+ */
+ private boolean isEmpty(int grpId, long pageId) {
+ return pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID;
+ }
+
+ /**
+ * Sets cell value to be empty.
+ *
+ * @param addr entry base address.
+ */
+ private void setEmpty(long addr) {
+ setPageId(addr, EMPTY_PAGE_ID);
+ setGrpId(addr, EMPTY_CACHE_GRP_ID);
+ setValue(addr, 0);
+ setIdealBucket(addr, 0);
+ setVersion(addr, 0);
+ }
+
+ /**
+ * @param base Entry base, address in buffer of the entry start.
+ * @param idxIdeal number of ideal bucket (cell) to insert this value.
+ */
+ private void setIdealBucket(long base, long idxIdeal) {
+ assert idxIdeal >= 0 && idxIdeal < numBuckets;
+
+ putLong(base + IDEAL_BUCKET_OFFSET, idxIdeal);
+ }
+
+ /**
+ * @return printable dump with all buckets state.
+ */
+ public String dump() {
+ StringBuilder sb = new StringBuilder();
+
+ for (int idx = 0; idx < numBuckets; idx++)
+ dumpEntry(sb, idx);
+
+ return sb.toString();
+ }
+
+ /**
+ * @param idx index of entry to dump
+ * @return string representation of bucket content.
+ */
+ private String dumpEntry(int idx) {
+ StringBuilder sb = new StringBuilder();
+
+ dumpEntry(sb, idx);
+
+ return sb.toString();
+ }
+
+ /**
+ * @param sb destination string builder to dump entry to.
+ * @param idx bucket index.
+ */
+ private void dumpEntry(StringBuilder sb, int idx) {
+ long base = entryBase(idx);
+ int curGrpId = getGrpId(base);
+ long curPageId = getPageId(base);
+ long curVal = getValue(base);
+ long ver = getVersion(base);
+
+ sb.append("slot [").append(idx).append("]:");
+
+ if (isEmpty(curGrpId, curPageId))
+ sb.append("Empty: ");
+
+ sb.append("i.buc=").append(getIdealBucket(base)).append(",");
+ sb.append("(grp=").append(curGrpId).append(",");
+ sb.append("page=").append(curPageId).append(")");
+ sb.append("->");
+ sb.append("(val=").append(curVal).append(",");
+ sb.append("ver=").append(ver).append(")");
+ sb.append("\n");
+ }
+
+ /**
+ * @param base Entry base, address in buffer of the entry start.
+ * @param idealBucket number of ideal bucket (cell) to insert this value.
+ * @param grpId Entry key. Group ID to be stored in entry.
+ * @param pageId Entry key. Page ID to be stored.
+ * @param val Entry value associated with key.
+ * @param ver Entry version.
+ */
+ private void setCellValue(long base, long idealBucket, int grpId, long pageId, long val, int ver) {
+ setIdealBucket(base, idealBucket);
+ setGrpId(base, grpId);
+ setPageId(base, pageId);
+ setValue(base, val);
+ setVersion(base, ver);
+ }
+
+ /**
+ * @param base address of current cell.
+ * @return number of ideal bucket (cell) to store this value.
+ */
+ private int getIdealBucket(long base) {
+ return getInt(base + IDEAL_BUCKET_OFFSET);
+ }
+
+ /**
+ * @param base Address of current cell.
+ * @return Page ID saved in cell.
+ */
+ private long getPageId(long base) {
+ return getLong(base + PAGE_ID_OFFSET);
+ }
+
+ /**
+ * @param base Address of cell.
+ * @param pageId Page ID to set in current cell.
+ */
+ private void setPageId(long base, long pageId) {
+ putLong(base + PAGE_ID_OFFSET, pageId);
+ }
+
+ /**
+ * @param base Entry base address.
+ * @return Group ID stored in entry.
+ */
+ private int getGrpId(long base) {
+ return getInt(base + GRP_ID_OFFSET);
+ }
+
+ /**
+ * @param base Entry base address.
+ * @param grpId Group ID to be stored in entry.
+ */
+ private void setGrpId(long base, int grpId) {
+ putInt(base + GRP_ID_OFFSET, grpId);
+ }
+
+ /**
+ * @param base Bucket base address.
+ * @return value stored in bucket.
+ */
+ private long getValue(long base) {
+ return getLong(base + VALUE_OFFSET);
+ }
+
+ /**
+ * @param base Bucket base address.
+ * @param val Value to store in bucket.
+ */
+ private void setValue(long base, long val) {
+ putLong(base + VALUE_OFFSET, val);
+ }
+
+ /**
+ * @param base Bucket base address.
+ * @return Entry version associated with bucket.
+ */
+ private int getVersion(long base) {
+ return getInt(base + VERSION_OFFSET);
+ }
+
+ /**
+ * @param base Bucket base address.
+ * @param ver Entry version to set in bucket.
+ */
+ private void setVersion(long base, int ver) {
+ putInt(base + VERSION_OFFSET, ver);
+ }
+
+ /** {@inheritDoc} */
+ @Override public final int size() {
+ return GridUnsafe.getInt(baseAddr + MAPSIZE_OFFSET);
+ }
+
+ /**
+ * Changes collection size.
+ *
+ * @param sz new size to set.
+ */
+ private void setSize(int sz) {
+ putInt(baseAddr + MAPSIZE_OFFSET, sz);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void forEach(BiConsumer<FullPageId, Long> act) {
+ for (int i = 0; i < numBuckets; i++) {
+ if (isEmptyAt(i))
+ continue;
+
+ long base = entryBase(i);
+
+ act.accept(getFullPageId(base), getValue(base));
+ }
+ }
+
+ /**
+ * @param base bucket base address.
+ * @return Key. Full page ID from bucket.
+ */
+ @NotNull private FullPageId getFullPageId(long base) {
+ return new FullPageId(getPageId(base), getGrpId(base));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
index 316cb8b..76cd5bd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
@@ -110,7 +110,7 @@ public class IgniteMassLoadSandboxTest extends GridCommonAbstractTest {
DataRegionConfiguration regCfg = new DataRegionConfiguration()
.setName("dfltMemPlc")
.setMetricsEnabled(true)
- .setMaxSize(256L * 1024 * 1024)
+ .setMaxSize(2 * 1024L * 1024 * 1024)
.setPersistenceEnabled(true);
DataStorageConfiguration dsCfg = new DataStorageConfiguration();
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
index 1f29549..43b27aa 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
@@ -20,32 +20,45 @@ package org.apache.ignite.internal.processors.cache.persistence.pagemem;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
import org.apache.ignite.internal.mem.DirectMemoryRegion;
import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.processors.cache.persistence.pagemem.FullPageIdTable;
-import org.apache.ignite.internal.util.typedef.CI2;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.logger.java.JavaLogger;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
/**
*
*/
-public class FullPageIdTableTest extends GridCommonAbstractTest {
+public class FullPageIdTableTest {
+ /** */
+ private static final int CACHE_ID_RANGE = 1;
+
+ /** */
+ private static final int PAGE_ID_RANGE = 3000;
+
/** */
- private static final int CACHE_ID_RANGE = 10;
+ private static final int CACHE_ID_RANGE2 = 1024;
/** */
- private static final int PAGE_ID_RANGE = 1000;
+ private static final int PAGE_ID_RANGE2 = 3000;
+
+ /** Logger. */
+ private JavaLogger log = new JavaLogger();
/**
* @throws Exception if failed.
*/
+ @Test
public void testRandomOperations() throws Exception {
- long mem = FullPageIdTable.requiredMemory(CACHE_ID_RANGE * PAGE_ID_RANGE);
+ int cnt = CACHE_ID_RANGE * PAGE_ID_RANGE;
+
+ long mem = FullPageIdTable.requiredMemory(cnt);
- UnsafeMemoryProvider prov = new UnsafeMemoryProvider(new JavaLogger());
+ UnsafeMemoryProvider prov = new UnsafeMemoryProvider(log);
prov.initialize(new long[] {mem});
@@ -58,7 +71,7 @@ public class FullPageIdTableTest extends GridCommonAbstractTest {
Random rnd = new Random(seed);
- FullPageIdTable tbl = new FullPageIdTable(region.address(), region.size(), true);
+ LoadedPagesMap tbl = new FullPageIdTable(region.address(), region.size(), true);
Map<FullPageId, Long> check = new HashMap<>();
@@ -77,7 +90,7 @@ public class FullPageIdTableTest extends GridCommonAbstractTest {
check.put(fullId, val);
}
else {
- tbl.remove(cacheId, pageId, 0);
+ tbl.remove(cacheId, pageId);
check.remove(fullId);
}
@@ -93,23 +106,163 @@ public class FullPageIdTableTest extends GridCommonAbstractTest {
}
/**
+ * @throws Exception if failed.
+ */
+ @Test
+ public void putRemoveScenario() throws Exception {
+ long seed = U.currentTimeMillis();
+
+ doPutRemoveTest(seed, false, 1_000_000);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ @Test
+ public void putRemoveScenarioNewMap() throws Exception {
+ long seed = U.currentTimeMillis();
+ doPutRemoveTest(seed, true, 30_000_000);
+ }
+
+ /**
+ * @param seed random initial value.
+ * @param newMapImpl use RobinHood map
+ * @param iters iterations.
+ */
+ private void doPutRemoveTest(long seed, boolean newMapImpl, int iters) {
+ int elementsCnt = 7000;
+
+ //System.setProperty(IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR, "11");
+ long mem = newMapImpl
+ ? RobinHoodBackwardShiftHashMap.requiredMemory(elementsCnt)
+ : FullPageIdTable.requiredMemory(elementsCnt);
+
+ DirectMemoryProvider prov = new UnsafeMemoryProvider(log);
+
+ prov.initialize(new long[] {mem});
+
+ DirectMemoryRegion region = prov.nextRegion();
+
+ try {
+ info("Seed: " + seed + "L; //");
+
+ Random rnd = new Random(seed);
+
+ LoadedPagesMap tbl =
+ newMapImpl
+ ? new RobinHoodBackwardShiftHashMap(region.address(), region.size())
+ : new FullPageIdTable(region.address(), region.size(), true);
+
+ Map<FullPageId, Long> check = new HashMap<>();
+
+ int tag = 0;
+ for (int i = 0; i < iters; i++) {
+ int op = rnd.nextInt(5);
+
+ int cacheId = rnd.nextInt(CACHE_ID_RANGE2) + 1;
+ int pageId = rnd.nextInt(PAGE_ID_RANGE2);
+
+ FullPageId fullId = new FullPageId(pageId, cacheId);
+
+ if (op == 0) {
+ long val = tbl.get(cacheId, pageId, tag, -1, -2);
+ if (val == -2)
+ tbl.refresh(cacheId, pageId, tag);
+ else {
+ Long checkVal = check.get(fullId);
+
+ if (checkVal != null) {
+ assertEquals("Ret " + val + "Check " + checkVal,
+ checkVal.longValue(), val);
+ }
+ }
+
+ }
+ else if ((op == 1 || op == 2) && (check.size() < elementsCnt)) {
+ long val = U.safeAbs(rnd.nextLong());
+
+ tbl.put(cacheId, pageId, val, tag);
+ check.put(fullId, val);
+ }
+ else if ((op == 3) && check.size() >= elementsCnt * 2 / 3) {
+ tbl.remove(cacheId, pageId);
+ check.remove(fullId);
+ }
+ else if (check.size() >= elementsCnt * 2 / 3) {
+ int idx = rnd.nextInt(tbl.capacity());
+ ReplaceCandidate ec = tbl.getNearestAt(idx);
+ if (ec != null) {
+ FullPageId fullPageId = ec.fullId();
+
+ tbl.remove(fullPageId.groupId(), fullPageId.pageId());
+
+ check.remove(fullPageId);
+ }
+ }
+
+ if (i > 0 && i % 100_000 == 0) {
+ info("Done: " + i
+ + " Size: " + check.size()
+ + " Capacity: " + tbl.capacity() );
+
+ verifyLinear(tbl, check);
+
+ tag++;
+ }
+
+ i++;
+ }
+
+ verifyLinear(tbl, check);
+ }
+ finally {
+ long msPassed = U.currentTimeMillis() - seed;
+ System.err.println("Seed used [" + seed + "] duration ["+ msPassed+ "] ms");
+ prov.shutdown();
+ }
+ }
+
+ /**
+ * @param msg Message to print.
+ */
+ protected void info(String msg) {
+ if (log.isInfoEnabled())
+ log.info(msg);
+
+ System.out.println(msg);
+ }
+
+
+ /**
* @param tbl Table to check.
* @param check Expected mapping.
*/
- private void verifyLinear(FullPageIdTable tbl, Map<FullPageId, Long> check) {
- final Map<FullPageId, Long> collector = new HashMap<>();
+ private void verifyLinear(LoadedPagesMap tbl, Map<FullPageId, Long> check) {
+ final Map<FullPageId, Long> tblSnapshot = new HashMap<>();
- tbl.visitAll(new CI2<FullPageId, Long>() {
- @Override public void apply(FullPageId fullId, Long val) {
- if (collector.put(fullId, val) != null)
- throw new AssertionError("Duplicate full page ID mapping: " + fullId);
- }
+ tbl.forEach((fullId, val) -> {
+ if (tblSnapshot.put(fullId, val) != null)
+ throw new AssertionError("Duplicate full page ID mapping: " + fullId);
});
- assertEquals("Size check failed", check.size(), collector.size());
+ int chkSize = check.size();
+ int foundTblSize = tblSnapshot.size();
- for (Map.Entry<FullPageId, Long> entry : check.entrySet())
- assertEquals("Mapping comparison failed for key: " + entry.getKey(),
- entry.getValue(), collector.get(entry.getKey()));
+ HashMap<FullPageId, Long> cp = new HashMap<>(tblSnapshot);
+ check.keySet().forEach(cp::remove);
+
+ assertEquals("Size check failed, check map size " +
+ chkSize + " but found in table " + foundTblSize + " elements," +
+ " table size " + tbl.size() +
+ "\n Difference: " + cp, chkSize, foundTblSize);
+
+ for (Map.Entry<FullPageId, Long> entry : check.entrySet()) {
+ Long valCheck = entry.getValue();
+ Long actual = tblSnapshot.get(entry.getKey());
+
+ if (!valCheck.equals(actual))
+ assertEquals("Mapping comparison failed for key: " + entry.getKey(),
+ valCheck, actual);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
new file mode 100644
index 0000000..da59145
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
@@ -0,0 +1,430 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.function.Consumer;
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.util.GridLongList;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+import org.jetbrains.annotations.NotNull;
+import org.junit.Test;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Unit tests of {@link RobinHoodBackwardShiftHashMap} implementation.
+ */
+public class RobinHoodBackwardShiftHashMapTest {
+
+ /**
+ * @param tester map test code
+ * @param cap required map capacity.
+ */
+ private void withMap(Consumer<RobinHoodBackwardShiftHashMap> tester, int cap) {
+ long memSize = RobinHoodBackwardShiftHashMap.requiredMemoryByBuckets(cap);
+ long addr = GridUnsafe.allocateMemory(memSize);
+
+ RobinHoodBackwardShiftHashMap map = new RobinHoodBackwardShiftHashMap(addr, memSize);
+ boolean success = false;
+ try {
+ tester.accept(map);
+
+ success = true;
+ }
+ finally {
+ if (!success)
+ System.err.println(map.dump());
+
+ GridUnsafe.freeMemory(addr);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testSimplestPutGet() throws Exception {
+ int cnt = 100;
+ withMap(map -> {
+ for (int i = 0; i < cnt; i++) {
+ int grpId = i + 1;
+ int val = grpId * grpId;
+
+ assertSizeChanged("Unique put should be successful " + grpId,
+ map, () -> map.put(grpId, 1, val, 1));
+ assertEquals(val, map.get(grpId, 1, 0, -1, -2));
+
+ assertSizeNotChanged("Duplicate put for " + grpId,
+ map, () -> map.put(grpId, 1, 1, 1));
+ assertEquals(1, map.get(grpId, 1, 0, -1, -2));
+ }
+
+ assertEquals(cnt, map.size());
+ }
+ , cnt);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test(expected = IgniteOutOfMemoryException.class)
+ public void testSimplestOverflow() throws Exception {
+ withMap(map -> {
+ for (int i = 0; i < 10; i++) {
+ int grpId = i + 1;
+ int val = grpId * grpId;
+ assertSizeChanged("Unique put should be successful [" + grpId + "]", map, () -> map.put(grpId, 1, val, 1));
+
+ assertEquals(val, map.get(grpId, 1, 0, -1, -2));
+
+ assertSizeNotChanged("Duplicate put for " + grpId, map, () -> map.put(grpId, 1, 1, 1));
+ assertEquals(1, map.get(grpId, 1, 0, -1, -2));
+ }
+
+ map.put(11, 1, 11, 1);
+ }
+ , 10);
+ }
+
+ /**
+ * @param msg message to dump in case assertion failed.
+ * @param map page map to check.
+ * @param act action during which size is expected to be changed.
+ */
+ private static void assertSizeChanged(String msg, LoadedPagesMap map, Runnable act) {
+ int size = map.size();
+ act.run();
+ int newSize = map.size();
+
+ assertNotEquals(msg, size, newSize);
+ }
+
+ /**
+ * @param msg message to dump in case assertion failed.
+ * @param map page map to check.
+ * @param act action during which size is expected to constant.
+ */
+ private static void assertSizeNotChanged(String msg, LoadedPagesMap map, Runnable act) {
+ int size = map.size();
+ act.run();
+ int newSize = map.size();
+
+ assertEquals(msg, size, newSize);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPutRemoveOnSamePlaces() throws Exception {
+ withMap(map -> {
+ doAddRemove(map);
+
+ //fill with 1 space left;
+ for (int i = 0; i < 99; i++) {
+ int grpId = i + 1;
+ int val = grpId * grpId;
+ assertSizeChanged("Unique put should be successful " + grpId, map,
+ () -> map.put(grpId, 1, val, 1));
+ }
+
+ doAddRemove(map);
+ }
+ , 100);
+ }
+
+ /**
+ * @param map tested map implementation
+ */
+ private void doAddRemove(LoadedPagesMap map) {
+ for (int i = 0; i < 100; i++) {
+ int grpId = i + 1;
+ int val = grpId * grpId;
+ map.put(grpId, 1, val, 1);
+ assertEquals(val, map.get(grpId, 1, 0, -1, -2));
+
+ assertTrue(map.remove(grpId, 1));
+ assertEquals(-1, map.get(grpId, 1, 0, -1, -2));
+ }
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void testCollisionOnRemove() {
+ Map<FullPageId, Long> ctrl = new LinkedHashMap<>();
+ int cap = 10;
+ FullPageId baseId = new FullPageId(0, 1);
+
+ withMap(map -> {
+ for (int i = 0; i < cap; i++) {
+ int grpId = i + 1;
+ int pageId = findPageIdForCollision(grpId, baseId, cap);
+ ctrl.put(new FullPageId(pageId, grpId), (long)grpId);
+ map.put(grpId, pageId, (long)grpId, 1);
+ }
+ for (FullPageId next : ctrl.keySet()) {
+ assertTrue(map.remove(next.groupId(), next.pageId()));
+ }
+ }, cap);
+ }
+
+ /**
+ * @param grpId Group ID to use
+ * @param id Page to be placed to same bucket with
+ * @param cap map maximum cells.
+ * @return page ID to use in addition to provided {@code grpId} to reach collision.
+ */
+ private int findPageIdForCollision(int grpId, FullPageId id, int cap) {
+ int bucket = U.safeAbs(id.hashCode()) % cap;
+
+ for (int p = 0; p < 1_000_000; p++) {
+ if (U.safeAbs(FullPageId.hashCode(grpId, p)) % cap == bucket)
+ return p;
+ }
+ assertTrue(false);
+ return -1;
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void testRandomOpsPutRemove() {
+ doPutRemoveTest(System.currentTimeMillis());
+ }
+
+ /**
+ * @param seed random seed, use timer to random run.
+ */
+ private void doPutRemoveTest(long seed) {
+ System.setProperty(IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR, "11");
+
+ int elementsCnt = 10_000;
+
+ withMap(tbl -> {
+ Random rnd = new Random(seed);
+
+ Map<FullPageId, Long> check = new HashMap<>();
+
+ int tag = 0;
+ for (int i = 0; i < 20_000_000; i++) {
+ int op = rnd.nextInt(5);
+
+ int cacheId = rnd.nextInt(100) + 1;
+ int pageId = rnd.nextInt(100);
+
+ FullPageId fullId = new FullPageId(pageId, cacheId);
+
+ if (op == 0) {
+ long val = tbl.get(cacheId, pageId, tag, -1, -2);
+ if (val == -2)
+ tbl.refresh(cacheId, pageId, tag);
+ else {
+ Long checkVal = check.get(fullId);
+
+ if (checkVal != null) {
+ assertEquals("Ret." +
+ getPageString(fullId) +
+ " tbl: " + val + " Check " + checkVal,
+ checkVal.longValue(), val);
+ }
+ }
+ }
+ else if ((op == 1 || op == 2) && (check.size() < elementsCnt)) {
+ long val = U.safeAbs(rnd.nextInt(30));
+
+ check.put(fullId, val);
+
+ tbl.put(cacheId, pageId, val, tag);
+ }
+ else if ((op == 3) && check.size() >= elementsCnt * 2 / 3) {
+ tbl.remove(cacheId, pageId);
+
+ check.remove(fullId);
+ }
+ else if (check.size() >= elementsCnt * 2 / 3) {
+ ReplaceCandidate ec = tbl.getNearestAt(rnd.nextInt(tbl.capacity()));
+
+ if (ec != null) {
+ FullPageId fullPageId = ec.fullId();
+
+ tbl.remove(fullPageId.groupId(), fullPageId.pageId());
+
+ check.remove(fullPageId);
+ }
+ }
+
+ i++;
+ }
+
+ }, elementsCnt);
+ }
+
+ /**
+ * @param fullId page ID.
+ * @return Printable string for page ID.
+ */
+ @NotNull private String getPageString(FullPageId fullId) {
+ return "(grp=" + fullId.groupId() + "," +
+ "page=" + fullId.pageId() + ")";
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPutAndCantGetOutdatedValue() throws Exception {
+ withMap(map -> {
+ //fill with 1 space left;
+ for (int i = 0; i < 99; i++) {
+ int ver = i;
+ int grpId = ver + 1;
+ int val = grpId * grpId;
+ map.put(grpId, 1, val, ver);
+
+ assertEquals(val, map.get(grpId, 1, ver, -1, -2));
+
+ assertEquals(-2, map.get(grpId, 1, ver + 1, -1, -2));
+ }
+ }, 100);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPutAndRefreshValue() throws Exception {
+ withMap(map -> {
+ //fill with 1 space left;
+ for (int i = 0; i < 99; i++) {
+ int ver = i;
+ int grpId = ver + 1;
+ int val = grpId * grpId;
+ int pageId = 1;
+ map.put(grpId, pageId, val, ver);
+
+ map.refresh(grpId, pageId, ver + 1);
+
+ assertEquals(val, map.get(grpId, pageId, ver + 1, -1, -2));
+
+ }
+
+ doAddRemove(map);
+ }, 100);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testClearAtWithControlMap3() throws Exception {
+ int cap = 100;
+
+ doRemovalTests(cap, (grpId, pageId) -> {
+ int hc = Integer.hashCode(grpId) + 31 * Long.hashCode(pageId);
+
+ return hc % 3 == 0;
+ });
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testClearAtWithControlMap7() throws Exception {
+ int cap = 100;
+
+ doRemovalTests(cap, (grpId, pageId) -> {
+ int hc = Integer.hashCode(grpId) + 31 * Long.hashCode(pageId);
+
+ return hc % 7 == 0;
+ });
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testClearAllWithControlMap() throws Exception {
+ int cap = 100;
+
+ doRemovalTests(cap, (grpId, pageId) -> true);
+ }
+
+ /**
+ * @param cap capacity of map
+ * @param pred predicate to filter entries to be removed.
+ */
+ private void doRemovalTests(int cap, LoadedPagesMap.KeyPredicate pred) {
+ withMap(map -> {
+ Map<FullPageId, Long> check = new HashMap<>();
+
+ int elems = cap - 1; //fill with 1 space left;
+ for (int i = 0; i < elems; i++) {
+ int grpId = i + 1;
+ long val = grpId * grpId;
+ int pageId = 1;
+
+ map.put(grpId, pageId, val, i);
+
+ check.put(new FullPageId(pageId, grpId), val);
+ }
+
+ int sz = map.size();
+
+ GridLongList list = map.removeIf(pred);
+ for (int i = 0; i < list.size(); i++) {
+ long val = list.get(i);
+
+ assertTrue(val > 0);
+ }
+
+ assertThat(sz - map.size(), Matchers.is(list.size()));
+
+ Map<FullPageId, Long> res = new HashMap<>();
+
+ map.forEach(res::put);
+
+ assertThat(map.size(), Matchers.is(map.size()));
+
+ for (int i = 0; i < cap; i++) {
+ GridLongList list1 = map.removeIf(i, i + 1, pred);
+
+ assertTrue(list1.isEmpty()); // clear should not be successful.
+ }
+
+ check.keySet().removeIf(entry -> pred.test(entry.groupId(), entry.pageId()));
+
+ MatcherAssert.assertThat(res, Matchers.is(check));
+
+ }, cap);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsUnitTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsUnitTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsUnitTestSuite.java
index 9906429..c022eea 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsUnitTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsUnitTestSuite.java
@@ -16,8 +16,10 @@
*/
package org.apache.ignite.testsuites;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.FullPageIdTableTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.IgnitePageMemReplaceDelayedWriteUnitTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.IgniteThrottlingUnitTest;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.RobinHoodBackwardShiftHashMapTest;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -27,7 +29,10 @@ import org.junit.runners.Suite;
@RunWith(Suite.class)
@Suite.SuiteClasses({
IgniteThrottlingUnitTest.class,
- IgnitePageMemReplaceDelayedWriteUnitTest.class
+ IgnitePageMemReplaceDelayedWriteUnitTest.class,
+ IgniteThrottlingUnitTest.class,
+ FullPageIdTableTest.class,
+ RobinHoodBackwardShiftHashMapTest.class
})
public class IgnitePdsUnitTestSuite {
}
[15/50] [abbrv] ignite git commit: IGNITE-7773 Add
'rebalanceClearingPartitionsLeft' metric to .NET - Fixes #3613.
Posted by sb...@apache.org.
IGNITE-7773 Add 'rebalanceClearingPartitionsLeft' metric to .NET - Fixes #3613.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/447a8a9d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/447a8a9d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/447a8a9d
Branch: refs/heads/ignite-zk
Commit: 447a8a9d0da87128b28d37f4ebaa2e3040e5febb
Parents: ed2435c
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Mon Mar 12 18:16:34 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Mar 12 18:16:34 2018 +0300
----------------------------------------------------------------------
.../internal/processors/platform/cache/PlatformCache.java | 1 +
.../dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs | 9 +++++++++
.../Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs | 7 +++++++
3 files changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/447a8a9d/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
index 86774c1..19dac83 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
@@ -1482,6 +1482,7 @@ public class PlatformCache extends PlatformAbstractTarget {
writer.writeLong(metrics.getHeapEntriesCount());
writer.writeLong(metrics.getEstimatedRebalancingFinishTime());
writer.writeLong(metrics.getRebalancingStartTime());
+ writer.writeLong(metrics.getRebalanceClearingPartitionsLeft());
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/447a8a9d/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
index 43a54c7..a328cf5 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
@@ -637,5 +637,14 @@ namespace Apache.Ignite.Core.Cache
/// Rebalancing start time.
/// </returns>
long RebalancingStartTime { get; }
+
+ /// <summary>
+ /// Gets number of partitions.
+ /// need to be cleared before actual rebalance start.
+ /// </summary>
+ /// <returns>
+ /// Number of clearing partitions for rebalance.
+ /// </returns>
+ long RebalanceClearingPartitionsLeft { get; }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/447a8a9d/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
index 1a3f0dc..fbc5d4c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
@@ -241,6 +241,9 @@ namespace Apache.Ignite.Core.Impl.Cache
/** */
private readonly long _rebalancingStartTime;
+ /** */
+ private readonly long _rebalancingClearingPartitionsLeft;
+
/// <summary>
/// Initializes a new instance of the <see cref="CacheMetricsImpl"/> class.
/// </summary>
@@ -319,6 +322,7 @@ namespace Apache.Ignite.Core.Impl.Cache
_heapEntriesCount = reader.ReadLong();
_estimatedRebalancingFinishTime = reader.ReadLong();
_rebalancingStartTime = reader.ReadLong();
+ _rebalancingClearingPartitionsLeft = reader.ReadLong();
}
/** <inheritDoc /> */
@@ -536,5 +540,8 @@ namespace Apache.Ignite.Core.Impl.Cache
/** <inheritDoc /> */
public long RebalancingStartTime { get { return _rebalancingStartTime; } }
+
+ /** <inheritDoc /> */
+ public long RebalanceClearingPartitionsLeft { get { return _rebalancingClearingPartitionsLeft; } }
}
}
\ No newline at end of file
[36/50] [abbrv] ignite git commit: IGNITE-7965 Robin-hood hashing may
fail with negative index in case backward shift finished with full table scan
- Fixes #3640.
Posted by sb...@apache.org.
IGNITE-7965 Robin-hood hashing may fail with negative index in case backward shift finished with full table scan - Fixes #3640.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/96d12d84
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/96d12d84
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/96d12d84
Branch: refs/heads/ignite-zk
Commit: 96d12d84bf1769418af4ff6c0324d8302a067535
Parents: 9fd6f4a
Author: dpavlov <dp...@gridgain.com>
Authored: Thu Mar 15 18:08:04 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Mar 15 18:08:04 2018 +0300
----------------------------------------------------------------------
.../pagemem/RobinHoodBackwardShiftHashMap.java | 5 ++++-
.../pagemem/RobinHoodBackwardShiftHashMapTest.java | 12 ++++++++++++
2 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/96d12d84/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
index fd4e0e6..e20673f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMap.java
@@ -147,7 +147,7 @@ public class RobinHoodBackwardShiftHashMap implements LoadedPagesMap {
* @return base cell (bucket) address in buffer.
*/
private long entryBase(int idx) {
- assert idx >= 0 && idx < numBuckets;
+ assert idx >= 0 && idx < numBuckets : "idx=" + idx + ", numBuckets=" + numBuckets;
return baseAddr + MAPSIZE_SIZE + MAPSIZE_PADDING + (long)idx * BYTES_PER_CELL;
}
@@ -320,6 +320,9 @@ public class RobinHoodBackwardShiftHashMap implements LoadedPagesMap {
int lastShiftedIdx = (idxRmv - 1) % numBuckets;
+ if (lastShiftedIdx < 0)
+ lastShiftedIdx += numBuckets;
+
setEmpty(entryBase(lastShiftedIdx));
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/96d12d84/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
index da59145..e594d06 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/RobinHoodBackwardShiftHashMapTest.java
@@ -69,6 +69,18 @@ public class RobinHoodBackwardShiftHashMapTest {
* @throws Exception If failed.
*/
@Test
+ public void testShortSize() throws Exception {
+ withMap(map -> {
+ map.put(1, 1, 0, 0);
+ map.put(2, 0, 1, 1);
+ map.remove(1, 1);
+ }, 2);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
public void testSimplestPutGet() throws Exception {
int cnt = 100;
withMap(map -> {
[41/50] [abbrv] ignite git commit: IGNITE-7890 Fixed error handing on
corrupted storage during node startup
Posted by sb...@apache.org.
IGNITE-7890 Fixed error handing on corrupted storage during node startup
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f623ef10
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f623ef10
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f623ef10
Branch: refs/heads/ignite-zk
Commit: f623ef10bb73a34ede5501eed78be362f3ba8c7d
Parents: 094f6c3
Author: Ilya Lantukh <il...@gridgain.com>
Authored: Fri Mar 16 14:25:13 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Mar 16 14:25:13 2018 +0300
----------------------------------------------------------------------
.../persistence/GridCacheOffheapManager.java | 9 ++
.../persistence/pagemem/PageMemoryImpl.java | 2 +-
.../IgnitePdsCorruptedStoreTest.java | 154 +++++++++++++++++++
.../ignite/testsuites/IgnitePdsTestSuite2.java | 3 +
4 files changed, 167 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f623ef10/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index d3b73bd..313a763 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -27,6 +27,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.NodeInvalidator;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -1179,6 +1180,14 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
delegate = delegate0;
}
+ catch (Throwable ex) {
+ U.error(log, "Unhandled exception during page store initialization. All further operations will " +
+ "be failed and local node will be stopped.", ex);
+
+ NodeInvalidator.INSTANCE.invalidate(ctx.kernalContext(), ex);
+
+ throw ex;
+ }
finally {
latch.countDown();
http://git-wip-us.apache.org/repos/asf/ignite/blob/f623ef10/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 9f979f5..5657099 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -897,7 +897,7 @@ public class PageMemoryImpl implements PageMemoryEx {
ByteBuffer restored = curPage == null ? lastValidPage : curPage;
if (restored == null)
- throw new AssertionError(String.format(
+ throw new IllegalStateException(String.format(
"Page is broken. Can't restore it from WAL. (grpId = %d, pageId = %X).",
fullId.groupId(), fullId.pageId()
));
http://git-wip-us.apache.org/repos/asf/ignite/blob/f623ef10/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
new file mode 100644
index 0000000..f49ebf6
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsCorruptedStoreTest extends GridCommonAbstractTest {
+ /** */
+ private static final String CACHE_NAME1 = "cache1";
+
+ /** */
+ private static final String CACHE_NAME2 = "cache2";
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ cleanPersistenceDir();
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+ cfg.setConsistentId(igniteInstanceName);
+
+ DataStorageConfiguration memCfg = new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setMaxSize(100 * 1024 * 1024)
+ .setPersistenceEnabled(true)
+ );
+
+ cfg.setDataStorageConfiguration(memCfg);
+
+ cfg.setCacheConfiguration(cacheConfiguration(CACHE_NAME1), cacheConfiguration(CACHE_NAME2));
+
+ return cfg;
+ }
+
+ /**
+ * @return File or folder in work directory.
+ * @throws IgniteCheckedException If failed to resolve file name.
+ */
+ private File file(String file) throws IgniteCheckedException {
+ return U.resolveWorkDirectory(U.defaultWorkDirectory(), file, false);
+ }
+
+ /**
+ * Create cache configuration.
+ *
+ * @param name Cache name.
+ */
+ private CacheConfiguration cacheConfiguration(String name) {
+ CacheConfiguration ccfg = new CacheConfiguration();
+ ccfg.setName(name);
+ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+ ccfg.setBackups(2);
+
+ return ccfg;
+ }
+
+ /**
+ * @throws Exception If test failed.
+ */
+ public void testNodeInvalidatedWhenPersistenceIsCorrupted() throws Exception {
+ Ignite ignite = startGrid(0);
+
+ startGrid(1);
+
+ ignite.cluster().active(true);
+
+ awaitPartitionMapExchange();
+
+ IgniteCache<Integer, String> cache1 = ignite.cache(CACHE_NAME1);
+
+ for (int i = 0; i < 100; ++i)
+ cache1.put(i, String.valueOf(i));
+
+ forceCheckpoint();
+
+ cache1.put(2, "test");
+
+ String nodeName = ignite.name().replaceAll("\\.", "_");
+
+ stopAllGrids();
+
+ U.delete(file(String.format("db/%s/cache-%s/part-2.bin", nodeName, CACHE_NAME1)));
+
+ ignite = startGrid(1);
+
+ try {
+ startGrid(0);
+ }
+ catch (IgniteCheckedException ex) {
+ // expected
+
+ throw ex;
+ }
+
+ IgniteCache<Integer, Integer> cache2 = ignite.cache(CACHE_NAME2);
+
+ // check that partition map exchange had been finished
+ cache2.put(1, 1);
+
+ assertEquals(1, ignite.cluster().nodes().size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f623ef10/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 5aca92f..f955b11 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -22,6 +22,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgniteDataStorage
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest2;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCorruptedStoreTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsExchangeDuringCheckpointTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
@@ -132,5 +133,7 @@ public class IgnitePdsTestSuite2 extends TestSuite {
suite.addTestSuite(WalCompactionTest.class);
suite.addTestSuite(IgniteCheckpointDirtyPagesForLowLoadTest.class);
+
+ suite.addTestSuite(IgnitePdsCorruptedStoreTest.class);
}
}
[37/50] [abbrv] ignite git commit: IGNITE-7800: Correction for
IgniteSystemProperties.IGNITE_NO_SHUTDOWN_HOOK javadoc. - Fixes #3644.
Posted by sb...@apache.org.
IGNITE-7800: Correction for IgniteSystemProperties.IGNITE_NO_SHUTDOWN_HOOK javadoc. - Fixes #3644.
Signed-off-by: shroman <rs...@yahoo.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/682197ba
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/682197ba
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/682197ba
Branch: refs/heads/ignite-zk
Commit: 682197ba952e1a011126071062b870fbbdd2f707
Parents: 96d12d8
Author: shroman <rs...@yahoo.com>
Authored: Fri Mar 16 12:13:10 2018 +0900
Committer: shroman <rs...@yahoo.com>
Committed: Fri Mar 16 12:13:10 2018 +0900
----------------------------------------------------------------------
.../src/main/java/org/apache/ignite/IgniteSystemProperties.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/682197ba/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 11419ec..cd62e48 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -65,7 +65,7 @@ public final class IgniteSystemProperties {
/** Defines Ignite installation folder. */
public static final String IGNITE_HOME = "IGNITE_HOME";
- /** If this system property is set to {@code false} - no shutdown hook will be set. */
+ /** If this system property is set to {@code true} - no shutdown hook will be set. */
public static final String IGNITE_NO_SHUTDOWN_HOOK = "IGNITE_NO_SHUTDOWN_HOOK";
/**
[34/50] [abbrv] ignite git commit: IGNITE-7958 Optimize on-focus-out
directive. 1. Don't parse expression every callback call. 2. Use $evalAsync
instead of $timeout.
Posted by sb...@apache.org.
IGNITE-7958 Optimize on-focus-out directive.
1. Don't parse expression every callback call.
2. Use $evalAsync instead of $timeout.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/22eda40c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/22eda40c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/22eda40c
Branch: refs/heads/ignite-zk
Commit: 22eda40c8d28ca056094d8d3338e5036e46c80d0
Parents: a69e1ac
Author: Ilya Borisov <kl...@gmail.com>
Authored: Thu Mar 15 10:10:17 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Mar 15 10:10:17 2018 +0700
----------------------------------------------------------------------
.../frontend/app/directives/on-focus-out.directive.js | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/22eda40c/modules/web-console/frontend/app/directives/on-focus-out.directive.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/directives/on-focus-out.directive.js b/modules/web-console/frontend/app/directives/on-focus-out.directive.js
index 2e6bfc6..802691f 100644
--- a/modules/web-console/frontend/app/directives/on-focus-out.directive.js
+++ b/modules/web-console/frontend/app/directives/on-focus-out.directive.js
@@ -15,15 +15,15 @@
* limitations under the License.
*/
-export default ['$timeout', '$parse', ($timeout, $parse) => {
+export default ['$parse', ($parse) => {
return ($scope, $element, $attrs) => {
+ const parsedExpr = $parse($attrs.igniteOnFocusOut);
+
const handlerCheckFocusOut = (FocusClick) => {
if ($element.find(FocusClick.target).length)
return;
- $parse($attrs.igniteOnFocusOut)($scope);
-
- $timeout();
+ $scope.$evalAsync(() => parsedExpr($scope));
};
window.addEventListener('click', handlerCheckFocusOut, true);
[48/50] [abbrv] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-zk
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index e848c37,7522881..663053f
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@@ -118,7 -118,7 +118,10 @@@ public enum GridTopic
TOPIC_INTERNAL_DIAGNOSTIC,
/** */
- TOPIC_WAL;
++ TOPIC_WAL,
++
++ /** */
+ TOPIC_METRICS;
/** Enum values. */
private static final GridTopic[] VALS = values();
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 3b320e5,0f952d9..7ee55f9
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@@ -882,16 -887,11 +889,21 @@@ public class GridIoMessageFactory imple
break;
case 129:
+ msg = new WalStateAckMessage();
+
+ break;
+
++ case 129:
+ msg = new ClusterMetricsUpdateMessage();
+
+ break;
+
+ case 130:
+ msg = new ContinuousRoutineStartResultMessage();
+
+ break;
+
- // [-3..119] [124..128] [-23..-27] [-36..-55]- this
+ // [-3..119] [124..129] [-23..-27] [-36..-55]- this
// [120..123] - DR
// [-4..-22, -30..-35] - SQL
// [2048..2053] - Snapshots
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 517a57d,e336667..24dfa2a
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@@ -53,17 -54,12 +54,15 @@@ import org.apache.ignite.cluster.Baseli
import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.CommunicationFailureResolver;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.DefaultCommunicationFailureResolver;
+import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
- import org.apache.ignite.internal.ClusterMetricsSnapshot;
import org.apache.ignite.internal.GridComponent;
import org.apache.ignite.internal.GridKernalContext;
- import org.apache.ignite.internal.GridNodeOrderComparator;
import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteKernal;
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
index d87eb8a,9402a32..38450df
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
@@@ -159,24 -159,17 +159,25 @@@ final class BinaryMetadataTransport
MetadataUpdateResultFuture resFut = new MetadataUpdateResultFuture();
if (log.isDebugEnabled())
- log.debug("Requesting metadata update for " + metadata.typeId());
+ log.debug("Requesting metadata update for " + metadata.typeId() + "; caller thread is blocked on future "
+ + resFut);
- synchronized (this) {
- unlabeledFutures.add(resFut);
+ try {
+ synchronized (this) {
+ unlabeledFutures.add(resFut);
- if (!stopping)
- discoMgr.sendCustomEvent(new MetadataUpdateProposedMessage(metadata, ctx.localNodeId()));
- else
- resFut.onDone(MetadataUpdateResult.createUpdateDisabledResult());
+ if (!stopping)
+ discoMgr.sendCustomEvent(new MetadataUpdateProposedMessage(metadata, ctx.localNodeId()));
+ else
+ resFut.onDone(MetadataUpdateResult.createUpdateDisabledResult());
+ }
}
+ catch (Exception e) {
+ resFut.onDone(MetadataUpdateResult.createUpdateDisabledResult(), e);
+ }
+
+ if (ctx.clientDisconnected())
+ onDisconnected();
return resFut;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 809a8c5,2d9a314..f0a5186
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@@ -55,8 -55,7 +55,9 @@@ import org.apache.ignite.cluster.Cluste
import org.apache.ignite.configuration.AddressResolver;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
+ import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.X;
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectApiExceptionTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
index 3fec03f,df93ae4..299dbf4
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
@@@ -113,11 -112,9 +112,11 @@@ public abstract class GridCacheAbstract
*
* @throws IgniteCheckedException If failed.
*/
- void initStoreStrategy() throws IgniteCheckedException {
+ protected void initStoreStrategy() throws IgniteCheckedException {
if (storeStgy == null)
storeStgy = isMultiJvm() ? new H2CacheStoreStrategy() : new MapCacheStoreStrategy();
+ else if (isMultiJvm() && !(storeStgy instanceof H2CacheStoreStrategy))
+ storeStgy = new H2CacheStoreStrategy();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index f5856d5,2337329..92577ad
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@@ -34,10 -35,9 +35,10 @@@ import org.apache.ignite.configuration.
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.internal.IgniteClientReconnectAbstractTest;
+ import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
- import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteClientDataStructuresAbstractTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 8eef43c,0704dbd..e456047
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@@ -52,9 -52,9 +52,10 @@@ import org.apache.ignite.configuration.
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.internal.DiscoverySpiTestListener;
import org.apache.ignite.internal.GridKernalContext;
- import org.apache.ignite.internal.GridNodeOrderComparator;
+ import org.apache.ignite.internal.cluster.NodeOrderComparator;
+ import org.apache.ignite.internal.cluster.NodeOrderLegacyComparator;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.IgniteNodeAttributes;
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/yardstick/pom-standalone.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/yardstick/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/23ac3166/modules/zookeeper/pom.xml
----------------------------------------------------------------------
[18/50] [abbrv] ignite git commit: IGNITE-7531: Data load benchmarks.
This closes #3571.
Posted by sb...@apache.org.
IGNITE-7531: Data load benchmarks. This closes #3571.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ac3d5450
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ac3d5450
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ac3d5450
Branch: refs/heads/ignite-zk
Commit: ac3d545045037446014b66108364bbef39362847
Parents: 1142a36
Author: Pavel Kuznetsov <pa...@gmail.com>
Authored: Tue Mar 13 15:46:36 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Tue Mar 13 15:46:36 2018 +0300
----------------------------------------------------------------------
.../ignite-localhost-persistence-config.xml | 71 +++++++
.../benchmark-jdbc-thin-inmemory.properties | 108 +++++++++++
.../benchmark-jdbc-thin-persistence.properties | 113 +++++++++++
.../yardstick/IgniteBenchmarkArguments.java | 9 +-
.../yardstick/jdbc/AbstractJdbcBenchmark.java | 23 ++-
.../upload/AbstractNativeBenchmark.java | 114 +++++++++++
.../upload/AbstractUploadBenchmark.java | 187 +++++++++++++++++++
.../upload/BatchedInsertBenchmark.java | 65 +++++++
.../ignite/yardstick/upload/CopyBenchmark.java | 125 +++++++++++++
.../yardstick/upload/InsertBenchmark.java | 52 ++++++
.../yardstick/upload/NativePutBenchmark.java | 39 ++++
.../upload/NativeStreamerBenchmark.java | 66 +++++++
.../upload/UploadBenchmarkArguments.java | 158 ++++++++++++++++
.../yardstick/upload/model/QueryFactory.java | 181 ++++++++++++++++++
.../ignite/yardstick/upload/model/Values10.java | 76 ++++++++
.../org/yardstickframework/package-info.java | 25 +++
.../probes/TotalTimeProbe.java | 137 ++++++++++++++
17 files changed, 1543 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/config/ignite-localhost-persistence-config.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-localhost-persistence-config.xml b/modules/yardstick/config/ignite-localhost-persistence-config.xml
new file mode 100644
index 0000000..2e9d9ff
--- /dev/null
+++ b/modules/yardstick/config/ignite-localhost-persistence-config.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!--
+ Ignite Spring configuration file to startup grid.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+ <import resource="ignite-base-config.xml"/>
+
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration" parent="base-ignite.cfg">
+ <property name="localHost" value="127.0.0.1"/>
+
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="addresses">
+ <list>
+ <value>127.0.0.1:47500</value>
+ <value>127.0.0.1:47501</value>
+ <value>127.0.0.1:47502</value>
+ <value>127.0.0.1:47503</value>
+ <value>127.0.0.1:47504</value>
+ <value>127.0.0.1:47505</value>
+ <value>127.0.0.1:47506</value>
+ <value>127.0.0.1:47507</value>
+ <value>127.0.0.1:47508</value>
+ <value>127.0.0.1:47509</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </property>
+
+ <property name="dataStorageConfiguration" >
+ <bean class="org.apache.ignite.configuration.DataStorageConfiguration">
+ <property name="defaultDataRegionConfiguration">
+ <bean class="org.apache.ignite.configuration.DataRegionConfiguration">
+ <property name="persistenceEnabled" value="true"/>
+ </bean>
+ </property>
+ </bean>
+ </property>
+
+ <property name="communicationSpi">
+ <bean class="org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi">
+ <property name="sharedMemoryPort" value="-1"/>
+ </bean>
+ </property>
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties b/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties
new file mode 100644
index 0000000..cacf726
--- /dev/null
+++ b/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Benchmarks for data upload in inmemory mode (persistence disabled).
+#
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
+-XX:+PrintGCDateStamps \
+"
+
+#Ignite version
+ver="RELEASE-"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
+BENCHMARK_DEFAULT_PROBES=TotalTimeProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+# Flag which indicates to restart the servers before every benchmark execution.
+RESTART_SERVERS=true
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# The benchmark is applicable only for 2 servers (the second server is started in client mode) and 1 driver.
+SERVER_HOSTS=localhost,localhost
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Backups count.
+b=1
+
+# Warmup.
+w=0
+
+# Threads count.
+t=1
+
+# Sync mode.
+sm=FULL_SYNC
+
+
+# Run configuration which contains all benchmarks.
+# Note that each benchmark is set to run only one time, warmup parameter is set to 0 due to custom warmup operation.
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn NativePutBenchmark -sn IgniteNode -ds ${ver}sql-upload-native-inmemory-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn NativeStreamerBenchmark -sn IgniteNode -ds ${ver}sql-upload-native-inmemory-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+ --streamer-local-batch-size 1000, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn CopyBenchmark -sn IgniteNode -ds ${ver}sql-upload-inmemory-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn InsertBenchmark -sn IgniteNode -ds ${ver}sql-upload-inmemory-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn BatchedInsertBenchmark -sn IgniteNode -ds ${ver}sql-upload-inmemory-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+ --upload-jdbc-batch-size 1000 \
+"
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties b/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties
new file mode 100644
index 0000000..b72e9b7
--- /dev/null
+++ b/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties
@@ -0,0 +1,113 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Benchmarks for data upload with enabled persistence.
+#
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
+-XX:+PrintGCDateStamps \
+"
+
+#Ignite version
+ver="RELEASE-"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
+BENCHMARK_DEFAULT_PROBES=TotalTimeProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+# Flag which indicates to restart the servers before every benchmark execution.
+RESTART_SERVERS=true
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# The benchmark is applicable only for 2 servers (the second server is started in client mode) and 1 driver.
+SERVER_HOSTS=localhost,localhost
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Backups count.
+b=1
+
+# Warmup.
+w=0
+
+# Threads count.
+t=1
+
+# Sync mode.
+sm=FULL_SYNC
+
+# Whither or not turn off WAL just for upload period.
+switchWal=false
+
+# Run configuration which contains all benchmarks.
+# Note that each benchmark is set to run only one time, warmup parameter is set to 0 due to custom warmup operation.
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn NativePutBenchmark -sn IgniteNode -ds ${ver}sql-upload-native-persistence-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn NativeStreamerBenchmark -sn IgniteNode -ds ${ver}sql-upload-native-persistence-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+ --streamer-local-batch-size 1000, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn CopyBenchmark -sn IgniteNode -ds ${ver}sql-upload-persistence-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+ --disable-wal ${switchWal}, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn InsertBenchmark -sn IgniteNode -ds ${ver}sql-upload-persistence-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+ --disable-wal ${switchWal}, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn BatchedInsertBenchmark -sn IgniteNode -ds ${ver}sql-upload-persistence-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+ --upload-jdbc-batch-size 1000 \
+ --disable-wal ${switchWal} \
+"
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
index 8c453ad..9789aea 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
@@ -18,6 +18,8 @@
package org.apache.ignite.yardstick;
import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParametersDelegate;
+import java.util.Collections;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.configuration.DataStorageConfiguration;
@@ -29,6 +31,7 @@ import org.apache.ignite.transactions.TransactionIsolation;
import java.util.ArrayList;
import java.util.List;
import org.apache.ignite.yardstick.cache.IgniteStreamerBenchmark;
+import org.apache.ignite.yardstick.upload.UploadBenchmarkArguments;
import org.jetbrains.annotations.Nullable;
/**
@@ -262,6 +265,10 @@ public class IgniteBenchmarkArguments {
@GridToStringInclude
private int clientNodesAfterId = -1;
+ @ParametersDelegate
+ @GridToStringInclude
+ public UploadBenchmarkArguments upload = new UploadBenchmarkArguments();
+
/**
* @return {@code True} if need set {@link DataStorageConfiguration}.
*/
@@ -384,7 +391,7 @@ public class IgniteBenchmarkArguments {
/**
* @return {@code True} if flag for native benchmarking is set.
*/
- public boolean isNative(){
+ public boolean isNative() {
return ntv;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
index 9ef8a77..6ab024f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
@@ -37,18 +37,18 @@ import static org.yardstickframework.BenchmarkUtils.println;
/**
* JDBC benchmark that performs query operations.
*/
-abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
+public abstract class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
/** All {@link Connection}s associated with threads. */
private final List<Connection> threadConnections = new ArrayList<>();
/** JDBC URL. */
- private String url;
+ protected String url;
/** Each connection is also a transaction, so we better pin them to threads. */
protected ThreadLocal<Connection> conn = new ThreadLocal<Connection>() {
@Override protected Connection initialValue() {
try {
- Connection conn = connection();
+ Connection conn = connection(url);
synchronized (threadConnections) {
threadConnections.add(conn);
@@ -66,6 +66,9 @@ abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
@Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
super.setUp(cfg);
+ // activate cluster if it is not auto activated
+ ignite().cluster().active(true);
+
if (url == null) {
if (args.jdbcUrl().startsWith(JdbcThinUtils.URL_PREFIX)) {
String addr = findThinAddress();
@@ -77,12 +80,22 @@ abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
println("Using jdbc url:" + url);
- fillData(cfg, (IgniteEx)ignite(), args.range());
+ setupData();
ignite().close();
}
/**
+ * Sets up test data
+ *
+ * Gets executed before local Ignite node is closed
+ * @throws Exception On error.
+ */
+ protected void setupData() throws Exception {
+ fillData(cfg, (IgniteEx)ignite(), args.range());
+ }
+
+ /**
* Find address of client node, that thin driver should use.
*
* @return Address for thin driver.
@@ -128,7 +141,7 @@ abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
* @return JDBC connection.
* @throws SQLException On error.
*/
- private Connection connection() throws SQLException {
+ protected final Connection connection(String url) throws SQLException {
println("JDBC connect to: " + url);
Connection conn = DriverManager.getConnection(url);
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
new file mode 100644
index 0000000..de1ad28
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.util.Map;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
+import org.apache.ignite.yardstick.upload.model.Values10;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Base class for benchmarks that perform upload using java api.
+ */
+public abstract class AbstractNativeBenchmark extends IgniteAbstractBenchmark {
+ /** Number of entries to be uploaded during warmup. */
+ private long insertRowsCnt;
+
+ /** Name of the {@link #cache} */
+ private String cacheName;
+
+ /** Cache method {@link test(Map)} uploads data to */
+ private IgniteCache<Long, Values10> cache;
+
+ /**
+ * Sets up benchmark: performs warmup on one cache and creates another for {@link #test(Map)} method.
+ *
+ * @param cfg Benchmark configuration.
+ * @throws Exception - on error.
+ */
+ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ cacheName = this.getClass().getSimpleName();
+
+ insertRowsCnt = args.upload.uploadRowsCnt();
+
+ // Number of entries to be uploaded during test().
+ long warmupRowsCnt = args.upload.warmupRowsCnt();
+
+ // warmup
+ BenchmarkUtils.println(cfg, "Starting custom warmup.");
+ String warmupCacheName = cacheName + "Warmup";
+
+ try (IgniteCache<Long, Values10> warmupCache = ignite().createCache(warmupCacheName)) {
+ upload(warmupCacheName, warmupRowsCnt);
+ }
+ finally {
+ ignite().destroyCache(warmupCacheName);
+ }
+
+ BenchmarkUtils.println(cfg, "Custom warmup finished.");
+
+ // cache for benchmarked action
+ cache = ignite().createCache(cacheName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void tearDown() throws Exception {
+ try {
+ if (cache == null)
+ throw new IllegalStateException("Cache is null, probably an error during setUp or warmup");
+
+ long size = cache.sizeLong();
+
+ if (size != insertRowsCnt) {
+ String msg = "Incorrect cache size: [actual=" + size + ", expected=" + insertRowsCnt +"].";
+
+ BenchmarkUtils.println(cfg, "TearDown: " + msg);
+
+ throw new RuntimeException(msg);
+ }
+
+ cache.close();
+
+ ignite().destroyCache(cacheName);
+
+ }
+ catch (IgniteException ex) {
+ BenchmarkUtils.println(cfg, "Could not close or destroy cache: " + ex);
+
+ throw ex;
+ }
+ finally {
+ super.tearDown();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+ upload(cacheName, insertRowsCnt);
+
+ return true;
+ }
+
+ /** Uploads {@param insertsCnt} to cache with name {@param cacheName} using java api. */
+ protected abstract void upload(String cacheName, long insertsCnt);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
new file mode 100644
index 0000000..81a93e2
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.yardstick.jdbc.AbstractJdbcBenchmark;
+import org.apache.ignite.yardstick.upload.model.QueryFactory;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Base class for upload benchmarks.
+ * Designed to run test method one single time.
+ * Introduces custom warmup operation.
+ */
+public abstract class AbstractUploadBenchmark extends AbstractJdbcBenchmark {
+ /** Total inserts size. */
+ long insertRowsCnt;
+
+ /** Rows count to be inserted and deleted during warmup */
+ long warmupRowsCnt;
+
+ /** Factory that hides all the test data details. */
+ protected QueryFactory queries = new QueryFactory();
+
+ /** {@inheritDoc} */
+ @Override public final void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ insertRowsCnt = args.upload.uploadRowsCnt();
+ warmupRowsCnt = args.upload.warmupRowsCnt();
+
+ init();
+
+ // Perform warmup keeping in mind wal optimization.
+ BenchmarkUtils.println(this.cfg, "Starting custom warmup");
+
+ if (args.upload.disableWal())
+ executeUpdate(QueryFactory.TURN_OFF_WAL);
+
+ try (Connection warmupConn = uploadConnection()) {
+ warmup(warmupConn);
+ }
+
+ if (args.upload.disableWal())
+ executeUpdate(QueryFactory.TURN_ON_WAL);
+
+ BenchmarkUtils.println(this.cfg, "Custom warmup finished");
+
+ dropAndCreate();
+ }
+
+ /**
+ * Inits benchmark fields.
+ */
+ protected void init() {
+ // No-op.
+ }
+
+ /**
+ * Method to warm up Benchmark server. <br/>
+ * In upload benchmarks we need warmup action
+ * and real test action to be separated.
+ */
+ protected abstract void warmup(Connection warmupConn) throws Exception;
+
+ /**
+ * Creates empty table.
+ */
+ @Override protected void setupData() throws Exception{
+ dropAndCreate();
+ }
+
+ /**
+ * Uploads data using this special connection, that may have additional
+ * url parameters, such as {@code streaming=true}.
+ */
+ protected abstract void upload(Connection uploadConn) throws Exception;
+
+ /** {@inheritDoc} */
+ @Override public final boolean test(Map<Object, Object> ctx) throws Exception {
+ if (args.upload.disableWal())
+ executeUpdate(QueryFactory.TURN_OFF_WAL);
+
+ try (Connection uploadConn = uploadConnection()) {
+ upload(uploadConn);
+ }
+
+ if (args.upload.disableWal())
+ executeUpdate(QueryFactory.TURN_ON_WAL);
+
+ return true;
+ }
+
+ /**
+ * Drops and re-creates test table.
+ */
+ private final void dropAndCreate() throws SQLException {
+ executeUpdate(QueryFactory.DROP_TABLE_IF_EXISTS);
+ executeUpdate(queries.createTable());
+ }
+
+ /**
+ * Retrieves records count in the test table.
+ */
+ public long count() throws SQLException {
+ try(PreparedStatement cnt = conn.get().prepareStatement(QueryFactory.COUNT)){
+ try (ResultSet rs = cnt.executeQuery()) {
+ rs.next();
+
+ return rs.getLong(1);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void tearDown() throws Exception {
+ BenchmarkUtils.println(cfg, "Tearing down");
+
+ try {
+ long cnt = count();
+
+ if (cnt != insertRowsCnt) {
+ String msg = "Rows count is incorrect: [actual=" + cnt + ", expected=" + insertRowsCnt + "]";
+
+ BenchmarkUtils.println(cfg, "TearDown: " + msg);
+
+ throw new RuntimeException(msg);
+ }
+
+ BenchmarkUtils.println(cfg, "Test table contains " + cnt + " rows.");
+ }
+ finally {
+ super.tearDown();
+ }
+
+ BenchmarkUtils.println(cfg, "TearDown successfully finished.");
+ }
+
+ /**
+ * Facility method for executing update queries.
+ */
+ private int executeUpdate(String updQry) throws SQLException {
+ try(PreparedStatement update = conn.get().prepareStatement(updQry)){
+ return update.executeUpdate();
+ }
+ }
+
+ /**
+ * Creates new connection only for upload purpose.
+ * This connection is special, since it may have additional jdbc url parameters.
+ */
+ private Connection uploadConnection() throws SQLException {
+ String urlParams = "";
+
+ // We can't just pass entire params string, due to yardstick, which relies on bash,
+ // has some troubles with escaping ampersand character.
+ List<String> rawParams = args.upload.uploadJdbcParams();
+
+ if (!rawParams.isEmpty()) {
+ String kvList = String.join("&", rawParams);
+ urlParams = "?" + kvList;
+ }
+
+ return connection(url + urlParams);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
new file mode 100644
index 0000000..a460e20
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Measures total time of data upload using batched insert.
+ * Supports streaming.
+ */
+public class BatchedInsertBenchmark extends AbstractUploadBenchmark {
+ /** Number of inserts in batch */
+ private long batchSize;
+
+ /** {@inheritDoc} */
+ @Override protected void init() {
+ batchSize = args.upload.jdbcBatchSize();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void warmup(Connection warmupConn) throws SQLException {
+ performBatchUpdate(warmupConn, warmupRowsCnt);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void upload(Connection uploadConn) throws Exception {
+ performBatchUpdate(uploadConn, insertRowsCnt);
+ }
+
+ /**
+ * Actually performs batched inserts, using specified connection.
+ *
+ * @param uploadConn Special connection for upload purposes.
+ * @param rowsCnt Number of rows to insert.
+ */
+ private void performBatchUpdate(Connection uploadConn, long rowsCnt) throws SQLException {
+ try (PreparedStatement insert = uploadConn.prepareStatement(queries.insert())) {
+ for (long id = 1; id <= rowsCnt; id++) {
+ queries.setRandomInsertArgs(insert, id);
+
+ insert.addBatch();
+
+ if (id % batchSize == 0 || id == rowsCnt)
+ insert.executeBatch();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
new file mode 100644
index 0000000..f0dcf20
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.io.File;
+import java.io.PrintStream;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Single shot benchmark for the COPY FROM... sql query.
+ * Measures total time of load number of rows from csv file using thin driver.
+ *
+ * Enabling streaming is useless for this benchmark, since copy uses streamer under the hood.
+ */
+public class CopyBenchmark extends AbstractUploadBenchmark {
+ /** csv file for warmup. */
+ private String warmupCsvPath;
+
+ /** csv file for benchmared action. */
+ private String realCsvPath;
+
+ /** packet_size parameter for sql COPY command.*/
+ private Long packetSize;
+
+ /** {@inheritDoc} */
+ @Override protected void init() {
+ super.init();
+
+ packetSize = args.upload.copyPacketSize();
+
+ warmupCsvPath = generateWarmupCsv();
+ realCsvPath = generateRealCsv();
+
+ }
+
+ /**
+ * Generate csv file for copy operation being benchmarked.
+ */
+ private String generateRealCsv() {
+ String prefix = "data-" + insertRowsCnt + "-rows-";
+
+ return generate(prefix, insertRowsCnt);
+ }
+
+ /**
+ * Generate csv file for copy operation being performed during warmup.
+ */
+ private String generateWarmupCsv() {
+ String prefix = "warmup-" + warmupRowsCnt + "-rows-" ;
+
+ return generate(prefix, warmupRowsCnt);
+ }
+
+ /**
+ * Generate csv file with specified prefix and containing specified number
+ * of records (lines).
+ * <br/>
+ * Each record contain incremental (starting 1) id and set of random values.
+ * <br/>
+ * File is created in current directory and is deleted on JVM exits.
+ *
+ * @param filePrefix prefix for csv to be generated.
+ * @param records how many csv lines generate.
+ *
+ * @return absolute path for generated csv file.
+ */
+ private String generate(String filePrefix, long records){
+ try {
+ File workDir = new File(System.getProperty("user.dir"));
+
+ File f = File.createTempFile(filePrefix, ".csv", workDir);
+
+ f.deleteOnExit();
+
+ BenchmarkUtils.println("Generating file: " + f.getAbsolutePath());
+
+ try (PrintStream out = new PrintStream(f)) {
+ for (long id = 1; id <= records; id++) {
+ String csvLine = queries.randomCsvLine(id);
+ out.println(csvLine);
+ }
+ }
+
+ long sizeMb = f.length() / (1024L * 1024L);
+
+ BenchmarkUtils.println("File have been generated (" + sizeMb + "MiB). It will be removed on exit");
+
+ return f.getAbsolutePath();
+ }
+ catch (Exception ex){
+ throw new RuntimeException("Couldn't generate CSV file, terminating", ex);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void warmup(Connection warmupConn) throws Exception {
+ try (PreparedStatement fromCsv = warmupConn.prepareStatement(queries.copyFrom(warmupCsvPath, packetSize))) {
+ fromCsv.executeUpdate();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void upload(Connection uploadConn) throws Exception {
+ try (PreparedStatement fromCsv = uploadConn.prepareStatement(queries.copyFrom(realCsvPath, packetSize))) {
+ fromCsv.executeUpdate();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
new file mode 100644
index 0000000..ecf5412
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Measures total time of upload data using sequence of single inserts;
+ * Supports streaming.
+ */
+public class InsertBenchmark extends AbstractUploadBenchmark {
+ /** {@inheritDoc} */
+ @Override public void warmup(Connection warmupConn) throws SQLException {
+ try (PreparedStatement insert = warmupConn.prepareStatement(queries.insert())) {
+ for (long id = 1; id <= warmupRowsCnt; id++) {
+ queries.setRandomInsertArgs(insert, id);
+
+ insert.executeUpdate();
+ }
+ }
+ }
+
+ /**
+ * Sequence of single inserts.
+ */
+ @Override public void upload (Connection uploadConn) throws Exception {
+ try (PreparedStatement insert = uploadConn.prepareStatement(queries.insert())) {
+ for (long id = 1; id <= insertRowsCnt; id++) {
+ queries.setRandomInsertArgs(insert, id);
+
+ insert.executeUpdate();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
new file mode 100644
index 0000000..60fbbda
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.yardstick.upload.model.Values10;
+
+/**
+ * Benchmark that inserts single upload of number of entries using {@link IgniteCache#put(Object, Object)}.
+ */
+public class NativePutBenchmark extends AbstractNativeBenchmark {
+ /**
+ * Uploads randomly generated data using simple put.
+ *
+ * @param cacheName - name of the cache.
+ * @param insertsCnt - how many entries should be uploaded.
+ */
+ @Override protected void upload(String cacheName, long insertsCnt) {
+ IgniteCache<Object, Object> c = ignite().cache(cacheName);
+
+ for (long id = 1; id <= insertsCnt; id++)
+ c.put(id, new Values10() );
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
new file mode 100644
index 0000000..e5117bc
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.yardstick.upload.model.Values10;
+
+/**
+ * Benchmark that performs single upload of number of entries using {@link IgniteDataStreamer}.
+ */
+public class NativeStreamerBenchmark extends AbstractNativeBenchmark {
+ /**
+ * Uploads randomly generated entries to specified cache.
+ *
+ * @param cacheName - name of the cache.
+ * @param insertsCnt - how many entries should be uploaded.
+ */
+ @Override protected void upload(String cacheName, long insertsCnt) {
+ try (IgniteDataStreamer<Long, Values10> streamer = ignite().dataStreamer(cacheName)) {
+ if (args.upload.streamerBufSize() != null)
+ streamer.perNodeBufferSize(args.upload.streamerBufSize());
+
+ if (args.upload.streamerNodeParOps() != null)
+ streamer.perNodeParallelOperations(args.upload.streamerNodeParOps());
+
+ int batchSize = args.upload.streamerLocBatchSize();
+
+ // IgniteDataStreamer.addData(Object, Object) has known performance issue,
+ // so we have an option to work it around.
+ if (batchSize == 1) {
+ for (long i = 1; i <= insertsCnt; i++)
+ streamer.addData(i, new Values10());
+ }
+ else {
+ Map<Long, Values10> buf = new HashMap<>(batchSize);
+
+ for (long i = 1; i <= insertsCnt; i++) {
+ buf.put(i, new Values10());
+
+ if (i % batchSize == 0 || i == insertsCnt) {
+ streamer.addData(buf);
+
+ buf.clear();
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
new file mode 100644
index 0000000..27a2164
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import com.beust.jcommander.Parameter;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
+import org.apache.ignite.yardstick.IgniteBenchmarkArguments;
+
+/**
+ * Represents command line arguments that are specific to upload benchmarks.
+ *
+ * @see IgniteBenchmarkArguments
+ */
+@SuppressWarnings({"UnusedDeclaration", "FieldCanBeLocal"})
+public class UploadBenchmarkArguments {
+ /** Whither or not temporary disable Write Ahead Log during upload. */
+ @Parameter(names = {"--disable-wal"},
+ arity = 1,
+ description = "Upload benchmark only: " +
+ "turn off Write Ahead Log before data uploading " +
+ "and turn it on again when upload is done.")
+ private boolean disableWal = false;
+
+ /**
+ * Parameters for JDBC connection, that only uploads data.
+ *
+ * We can't just pass entire params string, due to yardstick, which relies on bash,
+ * has some troubles with escaping ampersand character.
+ */
+ @Parameter(names = {"--sql-jdbc-params"},
+ variableArity = true,
+ description = "Upload benchmark only: " +
+ "Additional url parameters (space separated key=value) for special JDBC connection that only uploads data. ")
+ private List<String> uploadJdbcParams = Collections.emptyList();
+
+ @Parameter(names = {"--sql-copy-packet-size"},
+ description = "Upload benchmark only: use custom packet_size (in bytes) for copy command.")
+ private Long copyPacketSize = null;
+
+ @Parameter(names = {"--streamer-node-buf-size"},
+ description = "Native Streamer benchmark only: Set streamer's perNodeBufferSize property")
+ private Integer streamerBufSize = null;
+
+ @Parameter(names = {"--streamer-node-par-ops"},
+ description = "Native Streamer benchmark only: Set streamer's perNodeParallelOperations property")
+ private Integer streamerNodeParOps = null;
+
+ @Parameter(names = {"--streamer-local-batch-size"},
+ description = "Native Streamer benchmark only: collect entries to local map before passing to streamer." +
+ "If set to 1, than entries will be passed directly.")
+ private int streamerLocBatchSize = 1;
+
+ /** How many rows to upload during warmup. */
+ @Parameter(names = {"--upload-warmup-rows"})
+ private long warmupRowsCnt = 3_000_000;
+
+ /** How many rows to upload during real test. */
+ @Parameter(names = {"--upload-rows"})
+ private long uploadRowsCnt = -1;
+
+ /** How many rows to include in each batch ({@link BatchedInsertBenchmark} only). */
+ @Parameter(names = {"--upload-jdbc-batch-size"})
+ private long jdbcBatchSize = -1;
+
+ /**
+ * @return Switch wal.
+ */
+ public boolean disableWal() {
+ return disableWal;
+ }
+
+ /** @return parameters for JDBC url. */
+ public List<String> uploadJdbcParams() {
+ return uploadJdbcParams;
+ }
+
+ /** @return packet_size value for copy command or {@code null} for default value. */
+ @Nullable
+ public Long copyPacketSize() {
+ return copyPacketSize;
+ }
+
+ /**
+ * @return Value for {@link IgniteDataStreamer#perNodeBufferSize(int)}.
+ */
+ @Nullable
+ public Integer streamerBufSize() {
+ return streamerBufSize;
+ }
+
+ /**
+ * @return Value for {@link IgniteDataStreamer#perNodeParallelOperations(int)}.
+ */
+ @Nullable
+ public Integer streamerNodeParOps() {
+ return streamerNodeParOps;
+ }
+
+ /**
+ * How many entries to collect before passing map to {@link IgniteDataStreamer#addData(Map)}.
+ * If set to 1, {@link IgniteDataStreamer#addData(Object, Object)} method will be used.
+ */
+ public int streamerLocBatchSize() {
+ return streamerLocBatchSize;
+ }
+
+ /**
+ * See {@link #warmupRowsCnt}.
+ */
+ public long warmupRowsCnt() {
+ return warmupRowsCnt;
+ }
+
+ /**
+ * See {@link #uploadRowsCnt}.
+ */
+ public long uploadRowsCnt() {
+ if (uploadRowsCnt < 0)
+ throw new IllegalStateException("Upload rows count is not specified. Check arguments.");
+
+ return uploadRowsCnt;
+ }
+
+ /**
+ * See {@link #jdbcBatchSize}.
+ */
+ public long jdbcBatchSize() {
+ if (jdbcBatchSize < 0)
+ throw new IllegalStateException("JDBC batch size is not specified. Check arguments.");
+
+ return jdbcBatchSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return GridToStringBuilder.toString(UploadBenchmarkArguments.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
new file mode 100644
index 0000000..e14a091
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload.model;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.concurrent.ThreadLocalRandom;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Factory that hides all test data details:
+ * what query to use to create table
+ * or what random arguments to set in prepared statement.
+ */
+public class QueryFactory {
+ /** Query to drop table if it exists. */
+ public static final String DROP_TABLE_IF_EXISTS = "DROP TABLE IF EXISTS test_upload;";
+
+ /** Query to count table size. */
+ public static final String COUNT = "SELECT COUNT(id) FROM test_upload;";
+
+ /** Turns off Write Ahead Log. */
+ public static final String TURN_OFF_WAL = "ALTER TABLE test_upload NOLOGGING";
+
+ /** Turns on Write Ahead Log. */
+ public static final String TURN_ON_WAL = "ALTER TABLE test_upload LOGGING";
+
+ /** Number of "values" fields in the test table (any field except primary key). */
+ private int valFieldsCnt = 10;
+
+ /** Create table with long primary key and number of long and varchar fields */
+ private String createTable = newCreateTableQuery();
+
+ /** Parametrised query to insert new row. */
+ private String insert = newInsertQuery();
+
+ /**
+ * See {@link #createTable}.
+ */
+ private String newCreateTableQuery() {
+ StringBuilder create = new StringBuilder("CREATE TABLE test_upload (id LONG PRIMARY KEY");
+
+ for (int vi = 1; vi <= valFieldsCnt; vi++) {
+ create.append(", val_").append(vi);
+
+ if (vi % 2 == 1)
+ create.append(" VARCHAR(255)");
+ else
+ create.append(" LONG");
+
+ }
+
+ create.append(");");
+
+ return create.toString();
+ }
+
+ /**
+ * See {@link #insert}.
+ */
+ private String newInsertQuery() {
+ StringBuilder insert = new StringBuilder("INSERT INTO test_upload VALUES (?");
+ for (int vi = 1; vi <= valFieldsCnt; vi++)
+ insert.append(", ?");
+
+ insert.append(");");
+ return insert.toString();
+ }
+
+ /**
+ * See {@link #createTable}.
+ */
+ public String createTable() {
+ return createTable;
+ }
+
+ /**
+ * See {@link #insert}.
+ */
+ public String insert() {
+ return insert;
+ }
+
+ /**
+ * @param csvFilePath path to csv file.
+ * @param packetSize if not null, add packet_size query option.
+ * @return sql query that inserts data from specified csv file.
+ */
+ public String copyFrom(String csvFilePath, @Nullable Long packetSize) {
+ String pSizeExpr = "";
+
+ if (packetSize != null)
+ pSizeExpr = " packet_size " + packetSize;
+
+ return "COPY FROM \"" + csvFilePath + "\" " +
+ "INTO test_upload " + attributes() + " " +
+ "FORMAT CSV" + pSizeExpr + ";";
+ }
+
+ /**
+ * Creates string - comma-separated attributes of test table, surrounded with braces
+ * Is used as a part of sql statement.
+ *
+ * @return attributes list of test table as part of sql statement.
+ */
+ private String attributes() {
+ StringBuilder attrs = new StringBuilder("(id");
+
+ for (int vi = 1; vi <= valFieldsCnt; vi++)
+ attrs.append(", val_").append(vi);
+
+ attrs.append(')');
+
+ return attrs.toString();
+ }
+
+ /**
+ * Fills specified prepared statement with random values and specified id (primary key).
+ *
+ * @param stmt prepared statement, built from {@link #insert} query.
+ * @param id id in the test table.
+ * @throws SQLException if statement is not correct.
+ */
+ public void setRandomInsertArgs(PreparedStatement stmt, long id) throws SQLException {
+ stmt.setLong(1, id);
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int vi = 1; vi <= valFieldsCnt; vi++) {
+ // vi is value index (among all values), but we also have "id" which is primary key
+ // so index in query is value index shifted by 1
+ int qryIdx = vi + 1;
+
+ long nextVal = rnd.nextLong();
+
+ if (vi % 2 == 1)
+ stmt.setLong(qryIdx, nextVal);
+ else
+ stmt.setString(qryIdx, String.valueOf(nextVal));
+ }
+ }
+
+ /**
+ * Generates CSV line containing specified id and random values.
+ * This line corresponds 1 row of the test table,
+ * which will be inserted in the end.
+ *
+ * @param id key in the test table.
+ * @return generated comma-separated line.
+ */
+ public String randomCsvLine(long id) {
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ StringBuilder line = new StringBuilder().append(id);
+
+ for (int vi = 1; vi <= valFieldsCnt; vi++) {
+ line.append(',');
+
+ if (vi % 2 == 1)
+ line.append(rnd.nextLong());
+ else
+ line.append('"').append(rnd.nextLong()).append('"');
+ }
+
+ return line.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
new file mode 100644
index 0000000..f0bdee0
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload.model;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Describes data model.
+ * Matches data model, defined in {@link QueryFactory#createTable()}
+ */
+public class Values10 {
+ /** */
+ final String val1;
+
+ /** */
+ final long val2;
+
+ /** */
+ final String val3;
+
+ /** */
+ final long val4;
+
+ /** */
+ final String val5;
+
+ /** */
+ final long val6;
+
+ /** */
+ final String val7;
+
+ /** */
+ final long val8;
+
+ /** */
+ final String val9;
+
+ /** */
+ final long val10;
+
+ /** Creates new object with randomly initialized fields */
+ public Values10(){
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ val1 = String.valueOf(rnd.nextLong());
+ val2 = rnd.nextLong();
+
+ val3 = String.valueOf(rnd.nextLong());
+ val4 = rnd.nextLong();
+
+ val5 = String.valueOf(rnd.nextLong());
+ val6 = rnd.nextLong();
+
+ val7 = String.valueOf(rnd.nextLong());
+ val8 = rnd.nextLong();
+
+ val9 = String.valueOf(rnd.nextLong());
+ val10 = rnd.nextLong();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/yardstickframework/package-info.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/yardstickframework/package-info.java b/modules/yardstick/src/main/java/org/yardstickframework/package-info.java
new file mode 100644
index 0000000..989a667
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/yardstickframework/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Additional classes for yardstick framework
+ * This package name chosen so, according to default package, yardstick
+ * searches for classes to load
+ */
+
+package org.yardstickframework;
+
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java b/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
new file mode 100644
index 0000000..e4153ab
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.yardstickframework.probes;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkDriver;
+import org.yardstickframework.BenchmarkExecutionAwareProbe;
+import org.yardstickframework.BenchmarkProbePoint;
+import org.yardstickframework.BenchmarkTotalsOnlyProbe;
+
+import static org.yardstickframework.BenchmarkUtils.println;
+
+/**
+ * Probe that measures total execution time of action for each thread.
+ */
+public class TotalTimeProbe implements BenchmarkExecutionAwareProbe, BenchmarkTotalsOnlyProbe {
+ /** Thread local agents. */
+ private volatile ThreadAgent[] agents;
+
+ /** Benchmark configuration. */
+ private BenchmarkConfiguration cfg;
+
+ /** {@inheritDoc} */
+ @Override public void beforeExecute(int threadIdx) {
+ agents[threadIdx].before();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void afterExecute(int threadIdx) {
+ agents[threadIdx].after();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start(BenchmarkDriver drv, BenchmarkConfiguration cfg) throws Exception {
+ this.cfg = cfg;
+
+ agents = new ThreadAgent[cfg.threads()];
+
+ for (int thId = 0; thId < agents.length; thId++)
+ agents[thId] = new ThreadAgent();
+
+ println(cfg, "TotalTime probe started");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() throws Exception {
+ println(cfg, "TotalTime probe stopped");
+ }
+
+ /**
+ * This probe producing two-dimension points:
+ * Number of thread that executed {@code test} method and
+ * time it took to finish.
+ *
+ * @return meta information about each point.
+ */
+ @Override public Collection<String> metaInfo() {
+ return Arrays.asList("Thread ID", "Duration, ms");
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<BenchmarkProbePoint> points() {
+ Collection<BenchmarkProbePoint> points = new ArrayList<>();
+
+ for (int thId = 0; thId < agents.length; thId++) {
+
+ double durationMs = agents[thId].getDurationNs() / 1000_000d;
+
+ BenchmarkProbePoint pnt = new BenchmarkProbePoint(thId, new double[] {durationMs});
+
+ points.add(pnt);
+ }
+
+ return points;
+ }
+
+ /**
+ * Useless for {@link BenchmarkTotalsOnlyProbe}.
+ */
+ @Override public void buildPoint(long time) {
+ // No-op
+ }
+
+ /**
+ * Measures execution time. Used in thread local context.
+ */
+ static class ThreadAgent {
+ /** Start time of {@link org.yardstickframework.BenchmarkDriver#test(java.util.Map)} method. */
+ private volatile long startTs;
+
+ /** Finish time. */
+ private volatile long finishTs;
+
+ /**
+ * Start measuring.
+ */
+ public void before(){
+ startTs = System.nanoTime();
+ }
+
+ /**
+ * Stop measuring.
+ */
+ public void after(){
+ finishTs = System.nanoTime();
+ }
+
+ /**
+ * @return measured duration in nanoseconds.
+ */
+ public long getDurationNs(){
+ if (finishTs == 0L && startTs != 0)
+ throw new IllegalStateException("Cannot provide results of TotalTime probe; " +
+ "after() method haven't been called");
+
+ return (finishTs - startTs);
+ }
+ }
+}
[04/50] [abbrv] ignite git commit: IGNITE-7566 .NET: Build scripts:
stop build when one of the steps fails
Posted by sb...@apache.org.
IGNITE-7566 .NET: Build scripts: stop build when one of the steps fails
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/31087002
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/31087002
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/31087002
Branch: refs/heads/ignite-zk
Commit: 31087002b94127c3f1365e90e2a011b3585d085a
Parents: 0936f16
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Wed Mar 7 11:25:07 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Wed Mar 7 11:25:07 2018 +0300
----------------------------------------------------------------------
modules/platforms/dotnet/build.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/31087002/modules/platforms/dotnet/build.sh
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/build.sh b/modules/platforms/dotnet/build.sh
index 1a58ff0..66d3852 100755
--- a/modules/platforms/dotnet/build.sh
+++ b/modules/platforms/dotnet/build.sh
@@ -21,7 +21,7 @@
#
# Fail on error.
-#set -e
+set -e
# Build Java.
pushd .
[47/50] [abbrv] ignite git commit: IGNITE-7863: Spark dependencies
cleaned up. - Fixes #3648.
Posted by sb...@apache.org.
IGNITE-7863: Spark dependencies cleaned up. - Fixes #3648.
Signed-off-by: Nikolay Izhikov <ni...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/98ba7b32
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/98ba7b32
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/98ba7b32
Branch: refs/heads/ignite-zk
Commit: 98ba7b32decfc402deb34c9ed98ec19285fe5648
Parents: 892d8c7
Author: Nikolay Izhikov <ni...@apache.org>
Authored: Sat Mar 17 13:16:11 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Sat Mar 17 13:16:11 2018 +0300
----------------------------------------------------------------------
modules/spark-2.10/pom.xml | 119 -------------------------------
modules/spark/pom.xml | 152 ++--------------------------------------
2 files changed, 6 insertions(+), 265 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ba7b32/modules/spark-2.10/pom.xml
----------------------------------------------------------------------
diff --git a/modules/spark-2.10/pom.xml b/modules/spark-2.10/pom.xml
index f878c9e..1f40f0f 100644
--- a/modules/spark-2.10/pom.xml
+++ b/modules/spark-2.10/pom.xml
@@ -149,102 +149,11 @@
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-launcher_2.10</artifactId>
- <version>${spark.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
<artifactId>spark-tags_2.10</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>3.5</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- <version>4.0.29.Final</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.esotericsoftware.kryo</groupId>
- <artifactId>kryo</artifactId>
- <version>2.20</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>chill_2.10</artifactId>
- <version>0.8.1</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>3.0.2</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-json</artifactId>
- <version>3.1.2</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.tomcat</groupId>
- <artifactId>tomcat-servlet-api</artifactId>
- <version>8.0.23</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.glassfish.jersey.containers</groupId>
- <artifactId>jersey-container-servlet-core</artifactId>
- <version>2.25</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson2.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_2.10</artifactId>
- <version>${jackson2.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.xbean</groupId>
- <artifactId>xbean-asm5-shaded</artifactId>
- <version>4.5</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>net.jpountz.lz4</groupId>
- <artifactId>lz4</artifactId>
- <version>1.3.0</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-hadoop</artifactId>
<version>1.9.0</version>
@@ -257,34 +166,6 @@
</dependency>
<dependency>
- <groupId>org.codehaus.janino</groupId>
- <artifactId>janino</artifactId>
- <version>3.0.0</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.janino</groupId>
- <artifactId>commons-compiler</artifactId>
- <version>3.0.0</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.antlr</groupId>
- <artifactId>antlr4-runtime</artifactId>
- <version>4.5.3</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jul-to-slf4j</artifactId>
- <version>1.7.16</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava14.version}</version>
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ba7b32/modules/spark/pom.xml
----------------------------------------------------------------------
diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml
index 3c16996..e338c88 100644
--- a/modules/spark/pom.xml
+++ b/modules/spark/pom.xml
@@ -92,6 +92,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-tags_2.11</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${spark.hadoop.version}</version>
@@ -140,152 +146,6 @@
</exclusion>
</exclusions>
</dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-unsafe_2.11</artifactId>
- <version>${spark.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-launcher_2.11</artifactId>
- <version>${spark.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-tags_2.11</artifactId>
- <version>${spark.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>3.5</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- <version>4.0.29.Final</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.esotericsoftware.kryo</groupId>
- <artifactId>kryo</artifactId>
- <version>2.20</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>chill_2.11</artifactId>
- <version>0.8.1</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>3.0.2</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-json</artifactId>
- <version>3.1.2</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.tomcat</groupId>
- <artifactId>tomcat-servlet-api</artifactId>
- <version>8.0.23</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.glassfish.jersey.containers</groupId>
- <artifactId>jersey-container-servlet-core</artifactId>
- <version>2.25</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson2.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_2.11</artifactId>
- <version>${jackson2.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.xbean</groupId>
- <artifactId>xbean-asm5-shaded</artifactId>
- <version>4.5</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>net.jpountz.lz4</groupId>
- <artifactId>lz4</artifactId>
- <version>1.3.0</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-hadoop</artifactId>
- <version>1.9.0</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.janino</groupId>
- <artifactId>janino</artifactId>
- <version>3.0.0</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.codehaus.janino</groupId>
- <artifactId>commons-compiler</artifactId>
- <version>3.0.0</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.antlr</groupId>
- <artifactId>antlr4-runtime</artifactId>
- <version>4.5.3</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jul-to-slf4j</artifactId>
- <version>1.7.16</version>
- <scope>test</scope>
- </dependency>
</dependencies>
<build>
[20/50] [abbrv] ignite git commit: IGNITE-7638 Implemented robin-hood
hashing for FullPageIdTable - Fixes #3481.
Posted by sb...@apache.org.
IGNITE-7638 Implemented robin-hood hashing for FullPageIdTable - Fixes #3481.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1f8ec79d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1f8ec79d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1f8ec79d
Branch: refs/heads/ignite-zk
Commit: 1f8ec79d22305fb5a6e28c9c9fa8efa2a2e391bf
Parents: ac3d545
Author: dpavlov <dp...@gridgain.com>
Authored: Tue Mar 13 16:18:54 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Mar 13 16:18:54 2018 +0300
----------------------------------------------------------------------
modules/core/pom.xml | 7 +
.../apache/ignite/IgniteSystemProperties.java | 7 +
.../GridCacheDatabaseSharedManager.java | 18 +-
.../pagemem/CheckpointMetricsTracker.java | 2 +-
.../persistence/pagemem/EvictCandidate.java | 77 ---
.../persistence/pagemem/FullPageIdTable.java | 358 +++++-----
.../persistence/pagemem/LoadedPagesMap.java | 137 ++++
.../cache/persistence/pagemem/PageMemoryEx.java | 8 +-
.../persistence/pagemem/PageMemoryImpl.java | 192 +++---
.../persistence/pagemem/ReplaceCandidate.java | 78 +++
.../pagemem/RobinHoodBackwardShiftHashMap.java | 654 +++++++++++++++++++
.../checkpoint/IgniteMassLoadSandboxTest.java | 2 +-
.../pagemem/FullPageIdTableTest.java | 195 +++++-
.../RobinHoodBackwardShiftHashMapTest.java | 430 ++++++++++++
.../testsuites/IgnitePdsUnitTestSuite.java | 7 +-
15 files changed, 1812 insertions(+), 360 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/pom.xml
----------------------------------------------------------------------
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 68c19a7..e56d141 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -124,6 +124,13 @@
</dependency>
<dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-library</artifactId>
+ <version>1.3</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.5</version>
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index b1a1542..11419ec 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -815,12 +815,19 @@ public final class IgniteSystemProperties {
* Because other thread may require exactly the same page to be loaded from store, reads are protected by locking.
*/
public static final String IGNITE_DELAYED_REPLACED_PAGE_WRITE = "IGNITE_DELAYED_REPLACED_PAGE_WRITE";
+
/**
* When set to {@code true}, WAL implementation with dedicated worker will be used even in FSYNC mode.
* Default is {@code false}.
*/
public static final String IGNITE_WAL_FSYNC_WITH_DEDICATED_WORKER = "IGNITE_WAL_FSYNC_WITH_DEDICATED_WORKER";
+ /**
+ * When set to {@code false}, loaded pages implementation is switched to previous version of implementation,
+ * FullPageIdTable. {@code True} value enables 'Robin Hood hashing: backward shift deletion'.
+ * Default is {@code true}.
+ */
+ public static final String IGNITE_LOADED_PAGES_BACKWARD_SHIFT_MAP = "IGNITE_LOADED_PAGES_BACKWARD_SHIFT_MAP";
/**
* Enforces singleton.
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index d4ba4ef..d6a8a30 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -113,8 +113,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.StoredCacheData;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopologyImpl;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
@@ -141,7 +139,6 @@ import org.apache.ignite.internal.util.lang.GridInClosure3X;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.P3;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.LT;
@@ -1251,11 +1248,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
for (Map.Entry<PageMemoryEx, Collection<Integer>> entry : destroyed.entrySet()) {
final Collection<Integer> grpIds = entry.getValue();
- clearFuts.add(entry.getKey().clearAsync(new P3<Integer, Long, Integer>() {
- @Override public boolean apply(Integer grpId, Long pageId, Integer tag) {
- return grpIds.contains(grpId);
- }
- }, false));
+ clearFuts.add(entry.getKey().clearAsync((grpId, pageIdg) -> grpIds.contains(grpId), false));
}
for (IgniteInternalFuture<Void> clearFut : clearFuts) {
@@ -1985,12 +1978,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
PageMemoryEx pageMem = gId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(gId);
- pageMem.clearAsync(new P3<Integer, Long, Integer>() {
- @Override public boolean apply(Integer cacheId, Long pageId, Integer tag) {
- return cacheId == gId && PageIdUtils.partId(pageId) == pId;
- }
- }, true).get();
-
+ pageMem.clearAsync(
+ (grpId, pageId) -> grpId == gId && PageIdUtils.partId(pageId) == pId,
+ true).get();
}
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
index e533f41..dbe0f40 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
@@ -70,7 +70,7 @@ public class CheckpointMetricsTracker {
private long cpEnd;
/**
- *
+ * Increments counter if copy on write page was written.
*/
public void onCowPageWritten() {
COW_PAGES_UPDATER.incrementAndGet(this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/EvictCandidate.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/EvictCandidate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/EvictCandidate.java
deleted file mode 100644
index eb172c9..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/EvictCandidate.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence.pagemem;
-
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.util.tostring.GridToStringExclude;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- *
- */
-public class EvictCandidate {
- /** */
- private int tag;
-
- /** */
- @GridToStringExclude
- private long relPtr;
-
- /** */
- @GridToStringInclude
- private FullPageId fullId;
-
- /**
- * @param tag Tag.
- * @param relPtr Relative pointer.
- * @param fullId Full page ID.
- */
- public EvictCandidate(int tag, long relPtr, FullPageId fullId) {
- this.tag = tag;
- this.relPtr = relPtr;
- this.fullId = fullId;
- }
-
- /**
- * @return Tag.
- */
- public int tag() {
- return tag;
- }
-
- /**
- * @return Relative pointer.
- */
- public long relativePointer() {
- return relPtr;
- }
-
- /**
- * @return Index.
- */
- public FullPageId fullId() {
- return fullId;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(EvictCandidate.class, this, "relPtr", U.hexLong(relPtr));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java
index 19d26ff..7fabfe0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java
@@ -17,13 +17,14 @@
package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+import java.util.function.BiConsumer;
import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.util.GridLongList;
import org.apache.ignite.internal.util.GridUnsafe;
-import org.apache.ignite.internal.util.lang.GridPredicate3;
+import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiInClosure;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR;
import static org.apache.ignite.IgniteSystemProperties.getFloat;
@@ -31,7 +32,7 @@ import static org.apache.ignite.IgniteSystemProperties.getFloat;
/**
*
*/
-public class FullPageIdTable {
+public class FullPageIdTable implements LoadedPagesMap {
/** Load factor. */
private static final float LOAD_FACTOR = getFloat(IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR, 2.5f);
@@ -68,6 +69,15 @@ public class FullPageIdTable {
/** */
private static final int OUTDATED = -3;
+ /** Tag offset from entry base. */
+ private static final int TAG_OFFSET = 4;
+
+ /** Page id offset from entry base. */
+ private static final int PAGE_ID_OFFSET = 8;
+
+ /** Value offset from entry base. */
+ private static final int VALUE_OFFSET = 16;
+
/** Max size, in elements. */
protected int capacity;
@@ -102,31 +112,21 @@ public class FullPageIdTable {
clear();
}
- /**
- * @return Current number of entries in the map.
- */
- public final int size() {
+ /** {@inheritDoc} */
+ @Override public final int size() {
return GridUnsafe.getInt(valPtr);
}
- /**
- * @return Maximum number of entries in the map. This maximum can not be always reached.
- */
- public final int capacity() {
+ /** {@inheritDoc} */
+ @Override public final int capacity() {
return capacity;
}
- /**
- * Gets value associated with the given key.
- *
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @return A value associated with the given key.
- */
- public long get(int cacheId, long pageId, int tag, long absent, long outdated) {
- assert assertKey(cacheId, pageId);
+ /** {@inheritDoc} */
+ @Override public long get(int grpId, long pageId, int reqVer, long absent, long outdated) {
+ assert assertKey(grpId, pageId);
- int idx = getKey(cacheId, pageId, tag, false);
+ int idx = getKey(grpId, pageId, reqVer, false);
if (idx == -1)
return absent;
@@ -137,112 +137,97 @@ public class FullPageIdTable {
return valueAt(idx);
}
- /**
- * Refresh outdated value.
- *
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param tag Partition tag.
- * @return A value associated with the given key.
- */
- public long refresh(int cacheId, long pageId, int tag) {
- assert assertKey(cacheId, pageId);
+ /** {@inheritDoc} */
+ @Override public long refresh(int grpId, long pageId, int ver) {
+ assert assertKey(grpId, pageId);
- int idx = getKey(cacheId, pageId, tag, true);
+ int idx = getKey(grpId, pageId, ver, true);
- assert idx >= 0 : "[idx=" + idx + ", tag=" + tag + ", cacheId=" + cacheId +
- ", pageId=" + U.hexLong(pageId) + ']';
+ if (!(idx >= 0) || !(tagAt(idx) < ver)) {
+ A.ensure(idx >= 0, "[idx=" + idx + ", tag=" + ver + ", cacheId=" + grpId +
+ ", pageId=" + U.hexLong(pageId) + ']');
- assert tagAt(idx) < tag : "[idx=" + idx + ", tag=" + tag + ", cacheId=" + cacheId +
- ", pageId=" + U.hexLong(pageId) + ", tagAtIdx=" + tagAt(idx) + ']';
+ A.ensure(tagAt(idx) < ver, "[idx=" + idx + ", tag=" + ver + ", cacheId=" + grpId +
+ ", pageId=" + U.hexLong(pageId) + ", tagAtIdx=" + tagAt(idx) + ']');
+ }
- setTagAt(idx, tag);
+ setTagAt(idx, ver);
return valueAt(idx);
}
- /**
- * Associates the given key with the given value.
- * @param cacheId Cache ID
- * @param pageId Page ID.
- * @param value Value to set.
- */
- public void put(int cacheId, long pageId, long value, int tag) {
- assert assertKey(cacheId, pageId);
+ /** {@inheritDoc} */
+ @Override public void put(int grpId, long pageId, long val, int ver) {
+ assert assertKey(grpId, pageId);
- int index = putKey(cacheId, pageId, tag);
+ int idx = putKey(grpId, pageId, ver);
- setValueAt(index, value);
+ setValueAt(idx, val);
}
- /**
- * Removes key-value association for the given key.
- *
- * @param grpId Cache group ID.
- * @param pageId Page ID.
- */
- public void remove(int grpId, long pageId, int tag) {
+ /** {@inheritDoc} */
+ @Override public boolean remove(int grpId, long pageId) {
assert assertKey(grpId, pageId);
- int index = removeKey(grpId, pageId, tag);
+ int idx = removeKey(grpId, pageId);
+
+ boolean valRmv = idx >= 0;
+
+ if (valRmv)
+ setValueAt(idx, 0);
- if (index >= 0)
- setValueAt(index, 0);
+ return valRmv;
}
- /**
- * Find nearest value from specified position to the right.
- *
- * @param idx Index to start searching from.
- * @param absent Default value that will be returned if no values present.
- * @return Closest value to the index and it's partition tag or {@code absent} and -1 if no values found.
- */
- public EvictCandidate getNearestAt(final int idx, final long absent) {
- for (int i = idx; i < capacity + idx; i++) {
- final int idx2 = i >= capacity ? i - capacity : i;
+ /** {@inheritDoc} */
+ @Override public ReplaceCandidate getNearestAt(final int idxStart) {
+ for (int i = idxStart; i < capacity + idxStart; i++) {
+ final int idx2 = normalizeIndex(i);
if (isValuePresentAt(idx2)) {
long base = entryBase(idx2);
- int cacheId = GridUnsafe.getInt(base);
- int tag = GridUnsafe.getInt(base + 4);
- long pageId = GridUnsafe.getLong(base + 8);
- long val = GridUnsafe.getLong(base + 16);
+ int grpId = GridUnsafe.getInt(base);
+ int tag = GridUnsafe.getInt(base + TAG_OFFSET);
+ long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
+ long val = GridUnsafe.getLong(base + VALUE_OFFSET);
- return new EvictCandidate(tag, val, new FullPageId(pageId, cacheId));
+ return new ReplaceCandidate(tag, val, new FullPageId(pageId, grpId));
}
}
return null;
}
- /**
- * @param idx Index to clear value at.
- * @param pred Test predicate.
- * @param absent Value to return if the cell is empty.
- * @return Value at the given index.
- */
- public long clearAt(int idx, GridPredicate3<Integer, Long, Integer> pred, long absent) {
- long base = entryBase(idx);
+ /** {@inheritDoc} */
+ @Override public GridLongList removeIf(int startIdxToClear, int endIdxToClear, KeyPredicate keyPred) {
+ assert endIdxToClear > startIdxToClear
+ : "Start and end indexes are not consistent: {" + startIdxToClear + ", " + endIdxToClear + "}";
- int grpId = GridUnsafe.getInt(base);
- int tag = GridUnsafe.getInt(base + 4);
- long pageId = GridUnsafe.getLong(base + 8);
+ int sz = endIdxToClear - startIdxToClear;
- if ((pageId == REMOVED_PAGE_ID && grpId == REMOVED_CACHE_GRP_ID)
- || (pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID))
- return absent;
+ GridLongList list = new GridLongList(sz);
+
+ for (int idx = startIdxToClear; idx < endIdxToClear; idx++) {
+ long base = entryBase(idx);
+
+ int grpId = GridUnsafe.getInt(base);
+ long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
+
+ if (isRemoved(grpId, pageId) || isEmpty(grpId, pageId))
+ continue;
+
+ if (!keyPred.test(grpId, pageId))
+ continue;
- if (pred.apply(grpId, pageId, tag)) {
long res = valueAt(idx);
- setKeyAt(idx, REMOVED_CACHE_GRP_ID, REMOVED_PAGE_ID);
- setValueAt(idx, 0);
+ setRemoved(idx);
- return res;
+ list.add(res);
}
- else
- return absent;
+
+ return list;
}
/**
@@ -253,50 +238,50 @@ public class FullPageIdTable {
private int putKey(int cacheId, long pageId, int tag) {
int step = 1;
- int index = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
+ int idx = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
- int foundIndex = -1;
+ int foundIdx = -1;
int res;
do {
- res = testKeyAt(index, cacheId, pageId, tag);
+ res = testKeyAt(idx, cacheId, pageId, tag);
if (res == OUTDATED) {
- foundIndex = index;
+ foundIdx = idx;
break;
}
else if (res == EMPTY) {
- if (foundIndex == -1)
- foundIndex = index;
+ if (foundIdx == -1)
+ foundIdx = idx;
break;
}
else if (res == REMOVED) {
// Must continue search to the first empty slot to ensure there are no duplicate mappings.
- if (foundIndex == -1)
- foundIndex = index;
+ if (foundIdx == -1)
+ foundIdx = idx;
}
else if (res == EQUAL)
- return index;
+ return idx;
else
assert res == NOT_EQUAL;
- index++;
+ idx++;
- if (index >= capacity)
- index -= capacity;
+ if (idx >= capacity)
+ idx -= capacity;
}
while (++step <= maxSteps);
- if (foundIndex != -1) {
- setKeyAt(foundIndex, cacheId, pageId);
- setTagAt(foundIndex, tag);
+ if (foundIdx != -1) {
+ setKeyAt(foundIdx, cacheId, pageId);
+ setTagAt(foundIdx, tag);
if (res != OUTDATED)
incrementSize();
- return foundIndex;
+ return foundIdx;
}
throw new IgniteOutOfMemoryException("No room for a new key");
@@ -305,6 +290,7 @@ public class FullPageIdTable {
/**
* @param cacheId Cache ID.
* @param pageId Page ID.
+ * @param refresh Refresh.
* @return Key index.
*/
private int getKey(int cacheId, long pageId, int tag, boolean refresh) {
@@ -335,22 +321,97 @@ public class FullPageIdTable {
}
/**
+ * @param grpId Cache group ID.
+ * @param pageId Page ID.
+ * @return {@code true} if group & page id indicates cell has state 'Removed'.
+ */
+ private boolean isRemoved(int grpId, long pageId) {
+ return pageId == REMOVED_PAGE_ID && grpId == REMOVED_CACHE_GRP_ID;
+ }
+
+ /**
+ * @param idx cell index, normalized.
+ * @return {@code true} if cell with index idx has state 'Empty'.
+ */
+ private boolean isRemoved(int idx) {
+ long base = entryBase(idx);
+
+ int grpId = GridUnsafe.getInt(base);
+ long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
+
+ return isRemoved(grpId, pageId);
+ }
+
+ /**
+ * Sets cell state to 'Removed' or to 'Empty' if next cell is already 'Empty'.
+ * @param idx cell index, normalized.
+ */
+ private void setRemoved(int idx) {
+ setKeyAt(idx, REMOVED_CACHE_GRP_ID, REMOVED_PAGE_ID);
+
+ setValueAt(idx, 0);
+ }
+
+ /**
+ * @param grpId Cache group ID.
+ * @param pageId Page ID.
+ * @return {@code true} if group & page id indicates cell has state 'Empty'.
+ */
+ private boolean isEmpty(int grpId, long pageId) {
+ return pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID;
+ }
+
+ /**
+ * @param idx cell index, normalized.
+ * @return {@code true} if cell with index idx has state 'Empty'.
+ */
+ private boolean isEmpty(int idx) {
+ long base = entryBase(idx);
+
+ int grpId = GridUnsafe.getInt(base);
+ long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
+
+ return isEmpty(grpId, pageId);
+ }
+
+ /**
+ * Sets cell state to 'Empty'.
+ *
+ * @param idx cell index, normalized.
+ */
+ private void setEmpty(int idx) {
+ setKeyAt(idx, EMPTY_CACHE_GRP_ID, EMPTY_PAGE_ID);
+
+ setValueAt(idx, 0);
+ }
+
+ /**
+ * @param i index probably outsize internal array of cells.
+ * @return corresponding index inside cells array.
+ */
+ private int normalizeIndex(int i) {
+ assert i < 2 * capacity;
+
+ return i < capacity ? i : i - capacity;
+ }
+
+ /**
* @param cacheId Cache ID.
* @param pageId Page ID.
* @return Key index.
*/
- private int removeKey(int cacheId, long pageId, int tag) {
+ private int removeKey(int cacheId, long pageId) {
int step = 1;
- int index = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
+ int idx = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
- int foundIndex = -1;
+ int foundIdx = -1;
do {
- long res = testKeyAt(index, cacheId, pageId, tag);
+ long res = testKeyAt(idx, cacheId, pageId, -1);
if (res == EQUAL || res == OUTDATED) {
- foundIndex = index;
+ foundIdx = idx;
break;
}
@@ -359,20 +420,20 @@ public class FullPageIdTable {
else
assert res == REMOVED || res == NOT_EQUAL;
- index++;
+ idx++;
- if (index >= capacity)
- index -= capacity;
+ if (idx >= capacity)
+ idx -= capacity;
}
while (++step <= maxSteps);
- if (foundIndex != -1) {
- setKeyAt(foundIndex, REMOVED_CACHE_GRP_ID, REMOVED_PAGE_ID);
+ if (foundIdx != -1) {
+ setRemoved(foundIdx);
decrementSize();
}
- return foundIndex;
+ return foundIdx;
}
/**
@@ -383,16 +444,16 @@ public class FullPageIdTable {
long base = entryBase(index);
int grpId = GridUnsafe.getInt(base);
- int tag = GridUnsafe.getInt(base + 4);
- long pageId = GridUnsafe.getLong(base + 8);
+ int tag = GridUnsafe.getInt(base + TAG_OFFSET);
+ long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
- if (pageId == REMOVED_PAGE_ID && grpId == REMOVED_CACHE_GRP_ID)
+ if (isRemoved(grpId, pageId))
return REMOVED;
else if (pageId == testPageId && grpId == testCacheId && tag >= testTag)
return EQUAL;
else if (pageId == testPageId && grpId == testCacheId && tag < testTag)
return OUTDATED;
- else if (pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID)
+ else if (isEmpty(grpId, pageId))
return EMPTY;
else
return NOT_EQUAL;
@@ -406,10 +467,9 @@ public class FullPageIdTable {
long base = entryBase(idx);
int grpId = GridUnsafe.getInt(base);
- long pageId = GridUnsafe.getLong(base + 8);
+ long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
- return !((pageId == REMOVED_PAGE_ID && grpId == REMOVED_CACHE_GRP_ID)
- || (pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID));
+ return !isRemoved(grpId, pageId) && !isEmpty(grpId, pageId);
}
/**
@@ -425,15 +485,15 @@ public class FullPageIdTable {
}
/**
- * @param index Entry index.
+ * @param idx Entry index.
* @param grpId Cache group ID to write.
* @param pageId Page ID to write.
*/
- private void setKeyAt(int index, int grpId, long pageId) {
- long base = entryBase(index);
+ private void setKeyAt(int idx, int grpId, long pageId) {
+ long base = entryBase(idx);
GridUnsafe.putInt(base, grpId);
- GridUnsafe.putLong(base + 8, pageId);
+ GridUnsafe.putLong(base + PAGE_ID_OFFSET, pageId);
}
/**
@@ -478,21 +538,17 @@ public class FullPageIdTable {
return found ? scans : -scans;
}
- /**
- * Scans all the elements in this table.
- *
- * @param visitor Visitor.
- */
- public void visitAll(IgniteBiInClosure<FullPageId, Long> visitor) {
+ /** {@inheritDoc} */
+ @Override public void forEach(BiConsumer<FullPageId, Long> act) {
for (int i = 0; i < capacity; i++) {
if (isValuePresentAt(i)) {
long base = entryBase(i);
int cacheId = GridUnsafe.getInt(base);
- long pageId = GridUnsafe.getLong(base + 8);
- long val = GridUnsafe.getLong(base + 16);
+ long pageId = GridUnsafe.getLong(base + PAGE_ID_OFFSET);
+ long val = GridUnsafe.getLong(base + VALUE_OFFSET);
- visitor.apply(new FullPageId(pageId, cacheId), val);
+ act.accept(new FullPageId(pageId, cacheId), val);
}
}
}
@@ -502,7 +558,7 @@ public class FullPageIdTable {
* @return Value.
*/
private long valueAt(int index) {
- return GridUnsafe.getLong(entryBase(index) + 16);
+ return GridUnsafe.getLong(entryBase(index) + VALUE_OFFSET);
}
/**
@@ -510,27 +566,31 @@ public class FullPageIdTable {
* @param value Value.
*/
private void setValueAt(int index, long value) {
- GridUnsafe.putLong(entryBase(index) + 16, value);
+ GridUnsafe.putLong(entryBase(index) + VALUE_OFFSET, value);
}
- private long entryBase(int index) {
- return valPtr + 8 + (long)index * BYTES_PER_ENTRY;
+ /**
+ * @param idx Entry index.
+ * @return address of entry.
+ */
+ private long entryBase(int idx) {
+ return valPtr + 8 + (long)idx * BYTES_PER_ENTRY;
}
/**
- * @param index Index to get tag for.
+ * @param idx Index to get tag for.
* @return Tag at the given index.
*/
- private int tagAt(int index) {
- return GridUnsafe.getInt(entryBase(index) + 4);
+ private int tagAt(int idx) {
+ return GridUnsafe.getInt(entryBase(idx) + TAG_OFFSET);
}
/**
- * @param index Index to set tag for.
+ * @param idx Index to set tag for.
* @param tag Tag to set at the given index.
*/
- private void setTagAt(int index, int tag) {
- GridUnsafe.putInt(entryBase(index) + 4, tag);
+ private void setTagAt(int idx, int tag) {
+ GridUnsafe.putInt(entryBase(idx) + TAG_OFFSET, tag);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/LoadedPagesMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/LoadedPagesMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/LoadedPagesMap.java
new file mode 100644
index 0000000..1b23238
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/LoadedPagesMap.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.util.function.BiConsumer;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.util.GridLongList;
+
+/**
+ * Interface for storing correspondence of page ID in a cache group to long value (address in offheap segment). <br>
+ * Map support versioning of entries. Outdated entry (entry having version lower than requested),
+ * is not provided in case of get, outdated return value is provided instead.<br>
+ *
+ * This mapping is not thread safe. Operations should be protected by outside locking.<br>
+ */
+public interface LoadedPagesMap {
+ /**
+ * Gets value associated with the given key.
+ *
+ * @param grpId Cache Group ID. First part of the key.
+ * @param pageId Page ID. Second part of the key.
+ * @param reqVer Requested entry version, counter associated with value.
+ * @param absent return if provided page is not presented in map.
+ * @param outdated return if provided {@code reqVer} version is greater than value in map (was used for put).
+ * @return A value associated with the given key.
+ */
+ public long get(int grpId, long pageId, int reqVer, long absent, long outdated);
+
+ /**
+ * Associates the given key with the given value.
+ *
+ * @param grpId Cache Group ID. First part of the key.
+ * @param pageId Page ID. Second part of the key.
+ * @param val Value to set.
+ * @param ver Version/counter associated with value, can be used to check if value is outdated.
+ */
+ public void put(int grpId, long pageId, long val, int ver);
+
+ /**
+ * Refresh outdated value. Sets provided version to value associated with cache and page.
+ * Method should be called only for key present and only if version was outdated.
+ * Method may be called in case {@link #get(int, long, int, long, long)} returned {@code outdated} return value.
+ *
+ * @param grpId First part of the key. Cache Group ID.
+ * @param pageId Second part of the key. Page ID.
+ * @param ver Partition tag.
+ * @return A value associated with the given key.
+ * @throws IllegalArgumentException if method is called for absent key or key with fresh version.
+ */
+ public long refresh(int grpId, long pageId, int ver);
+
+ /**
+ * Removes key-value association for the given key.
+ * @param grpId First part of the key. Cache Group ID.
+ * @param pageId Second part of the key. Page ID.
+ * @return {@code True} if value was actually found and removed.
+ */
+ public boolean remove(int grpId, long pageId);
+
+ /**
+ * @return Maximum number of entries in the map. This maximum can not be always reached.
+ */
+ public int capacity();
+
+ /**
+ * @return Current number of entries in the map.
+ */
+ public int size();
+
+ /**
+ * Find nearest presented value from specified position to the right.
+ *
+ * @param idxStart Index to start searching from. Bounded with {@link #capacity()}.
+ * @return Closest value to the index and it's partition tag or {@code null} value that will
+ * be returned if no values present.
+ */
+ public ReplaceCandidate getNearestAt(int idxStart);
+
+
+ /**
+ * Removes entities matching provided predicate at specified mapping range.
+ *
+ * @param startIdxToClear Index to clear value at, inclusive. Bounded with {@link #capacity()}.
+ * @param endIdxToClear Index to clear value at, inclusive. Bounded with {@link #capacity()}.
+ * @param keyPred Test predicate for (cache group ID, page ID).
+ * @return List with removed values, value is not added to list for empty cell or if key is not matching to
+ * predicate.
+ */
+ public GridLongList removeIf(int startIdxToClear, int endIdxToClear, KeyPredicate keyPred);
+
+
+ /**
+ * Removes entities matching provided predicate.
+ *
+ * @param keyPred Test predicate for (cache group ID, page ID).
+ * @return List with removed values, value is not added to list for empty cell or if key is not matching to
+ * predicate.
+ */
+ default GridLongList removeIf(KeyPredicate keyPred) {
+ return removeIf(0, capacity(), keyPred);
+ }
+
+ /**
+ * Scans all the elements in this table.
+ *
+ * @param act Visitor/action to be applied to each not empty cell.
+ */
+ public void forEach(BiConsumer<FullPageId, Long> act);
+
+ /**
+ * Interface describing a predicate for Key (cache group ID, page ID). Usage of this predicate prevents odd object
+ * creation.
+ */
+ @FunctionalInterface public interface KeyPredicate {
+ /**
+ * Predicate body.
+ *
+ * @param grpId Cache group ID.
+ * @param pageId Page ID.
+ */
+ boolean test(int grpId, long pageId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
index 9c37508..e889fc5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
@@ -117,7 +117,7 @@ public interface PageMemoryEx extends PageMemory {
* the {@link #beginCheckpoint()} method call.
* @param outBuf Temporary buffer to write changes into.
* @param tracker Checkpoint metrics tracker.
- * @return {@code Partition tag} if data was read, {@code null} otherwise (data already saved to storage).
+ * @return {@code Partition generation} if data was read, {@code null} otherwise (data already saved to storage).
* @throws IgniteException If failed to obtain page data.
*/
@Nullable public Integer getForCheckpoint(FullPageId pageId, ByteBuffer outBuf, CheckpointMetricsTracker tracker);
@@ -127,7 +127,7 @@ public interface PageMemoryEx extends PageMemory {
*
* @param grpId Group ID.
* @param partId Partition ID.
- * @return New partition tag (growing 1-based partition file version).
+ * @return New partition generation (growing 1-based partition file version).
*/
public int invalidate(int grpId, int partId);
@@ -141,9 +141,9 @@ public interface PageMemoryEx extends PageMemory {
/**
* Asynchronously clears pages satisfying the given predicate.
*
- * @param pred Predicate for cache group id, pageId and partition tag.
+ * @param pred Predicate for cache group id, pageId.
* @param cleanDirty Flag indicating that dirty pages collection should be cleaned.
* @return Future that will be completed when all pages are cleared.
*/
- public IgniteInternalFuture<Void> clearAsync(GridPredicate3<Integer, Long, Integer> pred, boolean cleanDirty);
+ public IgniteInternalFuture<Void> clearAsync(LoadedPagesMap.KeyPredicate pred, boolean cleanDirty);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 27c9b38..fa10a1f 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -23,7 +23,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -38,6 +37,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.internal.IgniteInternalFuture;
@@ -62,6 +62,7 @@ import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockSta
import org.apache.ignite.internal.processors.cache.persistence.CheckpointWriteProgressSupplier;
import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionCountersIO;
@@ -76,9 +77,7 @@ import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.OffheapReadWriteLock;
import org.apache.ignite.internal.util.future.CountDownFuture;
import org.apache.ignite.internal.util.lang.GridInClosure3X;
-import org.apache.ignite.internal.util.lang.GridPredicate3;
import org.apache.ignite.internal.util.offheap.GridOffHeapOutOfMemoryException;
-import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
import org.jetbrains.annotations.Nullable;
@@ -157,7 +156,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/** Page lock offset. */
public static final int PAGE_LOCK_OFFSET = 32;
- /** Page lock offset. */
+ /** Page temp copy buffer relative pointer offset. */
public static final int PAGE_TMP_BUF_OFFSET = 40;
/**
@@ -189,6 +188,10 @@ public class PageMemoryImpl implements PageMemoryEx {
/** Number of used pages in checkpoint buffer. */
private final AtomicInteger cpBufPagesCntr = new AtomicInteger(0);
+ /** Use new implementation of loaded pages table: 'Robin Hood hashing: backward shift deletion'. */
+ private final boolean useBackwardShiftMap
+ = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_LOADED_PAGES_BACKWARD_SHIFT_MAP, true);
+
/** */
private ExecutorService asyncRunner = new ThreadPoolExecutor(
0,
@@ -483,7 +486,7 @@ public class PageMemoryImpl implements PageMemoryEx {
long relPtr = seg.loadedPages.get(
cacheId,
PageIdUtils.effectivePageId(pageId),
- seg.partTag(cacheId, partId),
+ seg.partGeneration(cacheId, partId),
INVALID_REL_PTR,
OUTDATED_REL_PTR
);
@@ -536,7 +539,7 @@ public class PageMemoryImpl implements PageMemoryEx {
}
}
- seg.loadedPages.put(cacheId, PageIdUtils.effectivePageId(pageId), relPtr, seg.partTag(cacheId, partId));
+ seg.loadedPages.put(cacheId, PageIdUtils.effectivePageId(pageId), relPtr, seg.partGeneration(cacheId, partId));
}
catch (IgniteOutOfMemoryException oom) {
DataRegionConfiguration dataRegionCfg = getDataRegionConfiguration();
@@ -628,7 +631,7 @@ public class PageMemoryImpl implements PageMemoryEx {
long relPtr = seg.loadedPages.get(
cacheId,
PageIdUtils.effectivePageId(pageId),
- seg.partTag(cacheId, partId),
+ seg.partGeneration(cacheId, partId),
INVALID_REL_PTR,
INVALID_REL_PTR
);
@@ -659,7 +662,7 @@ public class PageMemoryImpl implements PageMemoryEx {
long relPtr = seg.loadedPages.get(
cacheId,
PageIdUtils.effectivePageId(pageId),
- seg.partTag(cacheId, partId),
+ seg.partGeneration(cacheId, partId),
INVALID_REL_PTR,
OUTDATED_REL_PTR
);
@@ -688,7 +691,7 @@ public class PageMemoryImpl implements PageMemoryEx {
cacheId,
PageIdUtils.effectivePageId(pageId),
relPtr,
- seg.partTag(cacheId, partId)
+ seg.partGeneration(cacheId, partId)
);
long pageAddr = absPtr + PAGE_OVERHEAD;
@@ -788,7 +791,7 @@ public class PageMemoryImpl implements PageMemoryEx {
private long refreshOutdatedPage(Segment seg, int cacheId, long pageId, boolean rmv) {
assert seg.writeLock().isHeldByCurrentThread();
- int tag = seg.partTag(cacheId, PageIdUtils.partId(pageId));
+ int tag = seg.partGeneration(cacheId, PageIdUtils.partId(pageId));
long relPtr = seg.loadedPages.refresh(cacheId, PageIdUtils.effectivePageId(pageId), tag);
@@ -812,7 +815,7 @@ public class PageMemoryImpl implements PageMemoryEx {
}
if (rmv)
- seg.loadedPages.remove(cacheId, PageIdUtils.effectivePageId(pageId), tag);
+ seg.loadedPages.remove(cacheId, PageIdUtils.effectivePageId(pageId));
if (seg.segCheckpointPages != null)
seg.segCheckpointPages.remove(new FullPageId(pageId, cacheId));
@@ -1026,7 +1029,7 @@ public class PageMemoryImpl implements PageMemoryEx {
int tag;
- boolean tmpBuffer = false;
+ boolean pageSingleAcquire = false;
seg.readLock().lock();
@@ -1034,7 +1037,7 @@ public class PageMemoryImpl implements PageMemoryEx {
if (!isInCheckpoint(fullId))
return null;
- tag = seg.partTag(fullId.groupId(), PageIdUtils.partId(fullId.pageId()));
+ tag = seg.partGeneration(fullId.groupId(), PageIdUtils.partId(fullId.pageId()));
relPtr = seg.loadedPages.get(
fullId.groupId(),
@@ -1055,7 +1058,7 @@ public class PageMemoryImpl implements PageMemoryEx {
if (PageHeader.tempBufferPointer(absPtr) == INVALID_REL_PTR)
PageHeader.acquirePage(absPtr);
else
- tmpBuffer = true;
+ pageSingleAcquire = true;
}
}
finally {
@@ -1070,7 +1073,7 @@ public class PageMemoryImpl implements PageMemoryEx {
relPtr = seg.loadedPages.get(
fullId.groupId(),
PageIdUtils.effectivePageId(fullId.pageId()),
- seg.partTag(
+ seg.partGeneration(
fullId.groupId(),
PageIdUtils.partId(fullId.pageId())
),
@@ -1099,19 +1102,22 @@ public class PageMemoryImpl implements PageMemoryEx {
}
}
else
- return copyPageForCheckpoint(absPtr, fullId, outBuf, tmpBuffer, tracker) ? tag : null;
+ return copyPageForCheckpoint(absPtr, fullId, outBuf, pageSingleAcquire, tracker) ? tag : null;
}
/**
* @param absPtr Absolute ptr.
* @param fullId Full id.
- * @param tmpBuf Tmp buffer.
+ * @param outBuf Output buffer to write page content into.
+ * @param pageSingleAcquire Page is acquired only once. We don't pin the page second time (until page will not be
+ * copied) in case checkpoint temporary buffer is used.
+ * @param tracker Checkpoint statistics tracker.
*/
private boolean copyPageForCheckpoint(
long absPtr,
FullPageId fullId,
- ByteBuffer tmpBuf,
- boolean tmpBuffer,
+ ByteBuffer outBuf,
+ boolean pageSingleAcquire,
CheckpointMetricsTracker tracker
) {
assert absPtr != 0;
@@ -1131,7 +1137,7 @@ public class PageMemoryImpl implements PageMemoryEx {
long tmpAbsPtr = checkpointPool.absolute(tmpRelPtr);
- copyInBuffer(tmpAbsPtr, tmpBuf);
+ copyInBuffer(tmpAbsPtr, outBuf);
GridUnsafe.setMemory(tmpAbsPtr + PAGE_OVERHEAD, pageSize(), (byte)0);
@@ -1145,12 +1151,12 @@ public class PageMemoryImpl implements PageMemoryEx {
// Need release again because we pin page when resolve abs pointer,
// and page did not have tmp buffer page.
- if (!tmpBuffer)
+ if (!pageSingleAcquire)
PageHeader.releasePage(absPtr);
}
else {
- copyInBuffer(absPtr, tmpBuf);
+ copyInBuffer(absPtr, outBuf);
PageHeader.dirty(absPtr, false);
@@ -1158,8 +1164,8 @@ public class PageMemoryImpl implements PageMemoryEx {
PageHeader.releasePage(absPtr);
}
- assert PageIO.getType(tmpBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
- assert PageIO.getVersion(tmpBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(fullId.pageId());
+ assert PageIO.getType(outBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
+ assert PageIO.getVersion(outBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(fullId.pageId());
return true;
}
@@ -1199,7 +1205,7 @@ public class PageMemoryImpl implements PageMemoryEx {
seg.writeLock().lock();
try {
- int newTag = seg.incrementPartTag(grpId, partId);
+ int newTag = seg.incrementPartGeneration(grpId, partId);
if (tag == 0)
tag = newTag;
@@ -1220,7 +1226,7 @@ public class PageMemoryImpl implements PageMemoryEx {
seg.writeLock().lock();
try {
- seg.resetPartTags(grpId);
+ seg.resetGroupPartitionsGeneration(grpId);
}
finally {
seg.writeLock().unlock();
@@ -1230,7 +1236,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@Override public IgniteInternalFuture<Void> clearAsync(
- GridPredicate3<Integer, Long, Integer> pred,
+ LoadedPagesMap.KeyPredicate pred,
boolean cleanDirty
) {
CountDownFuture completeFut = new CountDownFuture(segments.length);
@@ -1811,16 +1817,22 @@ public class PageMemoryImpl implements PageMemoryEx {
/** */
private static final double FULL_SCAN_THRESHOLD = 0.4;
+ /** Pointer to acquired pages integer counter. */
+ private static final int ACQUIRED_PAGES_SIZEOF = 4;
+
+ /** Padding to read from word beginning. */
+ private static final int ACQUIRED_PAGES_PADDING = 4;
+
/** Page ID to relative pointer map. */
- private FullPageIdTable loadedPages;
+ private LoadedPagesMap loadedPages;
- /** */
+ /** Pointer to acquired pages integer counter. */
private long acquiredPagesPtr;
/** */
private PagePool pool;
- /** */
+ /** Bytes required to store {@link #loadedPages}. */
private long memPerTbl;
/** Pages marked as dirty since the last checkpoint. */
@@ -1832,8 +1844,11 @@ public class PageMemoryImpl implements PageMemoryEx {
/** */
private final int maxDirtyPages;
- /** Maps partition (cacheId, partId) to its tag. Tag is 1-based incrementing partition file counter */
- private final Map<T2<Integer, Integer>, Integer> partTagMap = new HashMap<>();
+ /** Initial partition generation. */
+ private static final int INIT_PART_GENERATION = 1;
+
+ /** Maps partition (cacheId, partId) to its generation. Generation is 1-based incrementing partition counter. */
+ private final Map<GroupPartitionId, Integer> partGenerationMap = new HashMap<>();
/** */
private boolean closed;
@@ -1847,15 +1862,23 @@ public class PageMemoryImpl implements PageMemoryEx {
int pages = (int)(totalMemory / sysPageSize);
- memPerTbl = requiredSegmentTableMemory(pages);
-
acquiredPagesPtr = region.address();
GridUnsafe.putIntVolatile(null, acquiredPagesPtr, 0);
- loadedPages = new FullPageIdTable(region.address() + 8, memPerTbl, true);
+ int ldPagesMapOffInRegion = ACQUIRED_PAGES_SIZEOF + ACQUIRED_PAGES_PADDING;
+
+ long ldPagesAddr = region.address() + ldPagesMapOffInRegion;
+
+ memPerTbl = useBackwardShiftMap
+ ? RobinHoodBackwardShiftHashMap.requiredMemory(pages)
+ : requiredSegmentTableMemory(pages);
- DirectMemoryRegion poolRegion = region.slice(memPerTbl + 8);
+ loadedPages = useBackwardShiftMap
+ ? new RobinHoodBackwardShiftHashMap(ldPagesAddr, memPerTbl)
+ : new FullPageIdTable(ldPagesAddr, memPerTbl, true);
+
+ DirectMemoryRegion poolRegion = region.slice(memPerTbl + ldPagesMapOffInRegion);
pool = new PagePool(idx, poolRegion, null);
@@ -1980,7 +2003,7 @@ public class PageMemoryImpl implements PageMemoryEx {
saveDirtyPage.writePage(
fullPageId,
wrapPointer(absPtr + PAGE_OVERHEAD, pageSize()),
- partTag(
+ partGeneration(
fullPageId.groupId(),
PageIdUtils.partId(fullPageId.pageId())
)
@@ -2102,13 +2125,13 @@ public class PageMemoryImpl implements PageMemoryEx {
// We need to lookup for pages only in current segment for thread safety,
// so peeking random memory will lead to checking for found page segment.
// It's much faster to check available pages for segment right away.
- EvictCandidate nearest = loadedPages.getNearestAt(rnd.nextInt(cap), INVALID_REL_PTR);
+ ReplaceCandidate nearest = loadedPages.getNearestAt(rnd.nextInt(cap));
assert nearest != null && nearest.relativePointer() != INVALID_REL_PTR;
long rndAddr = nearest.relativePointer();
- int partTag = nearest.tag();
+ int partGen = nearest.generation();
final long absPageAddr = absolute(rndAddr);
@@ -2118,7 +2141,7 @@ public class PageMemoryImpl implements PageMemoryEx {
assert fullId.equals(nearest.fullId()) : "Invalid page mapping [tableId=" + nearest.fullId() +
", actual=" + fullId + ", nearest=" + nearest;
- boolean outdated = partTag < partTag(fullId.groupId(), PageIdUtils.partId(fullId.pageId()));
+ boolean outdated = partGen < partGeneration(fullId.groupId(), PageIdUtils.partId(fullId.pageId()));
if (outdated)
return refreshOutdatedPage(this, fullId.groupId(), fullId.pageId(), true);
@@ -2184,11 +2207,7 @@ public class PageMemoryImpl implements PageMemoryEx {
loadedPages.remove(
fullPageId.groupId(),
- PageIdUtils.effectivePageId(fullPageId.pageId()),
- partTag(
- fullPageId.groupId(),
- PageIdUtils.partId(fullPageId.pageId())
- )
+ PageIdUtils.effectivePageId(fullPageId.pageId())
);
return relRmvAddr;
@@ -2231,19 +2250,19 @@ public class PageMemoryImpl implements PageMemoryEx {
int failToPrepare = 0;
for (int i = 0; i < cap; i++) {
- final EvictCandidate nearest = loadedPages.getNearestAt(i, INVALID_REL_PTR);
+ final ReplaceCandidate nearest = loadedPages.getNearestAt(i);
assert nearest != null && nearest.relativePointer() != INVALID_REL_PTR;
final long addr = nearest.relativePointer();
- int partTag = nearest.tag();
+ int partGen = nearest.generation();
final long absPageAddr = absolute(addr);
FullPageId fullId = PageHeader.fullPageId(absPageAddr);
- if (partTag < partTag(fullId.groupId(), PageIdUtils.partId(fullId.pageId())))
+ if (partGen < partGeneration(fullId.groupId(), PageIdUtils.partId(fullId.pageId())))
return refreshOutdatedPage(this, fullId.groupId(), fullId.pageId(), true);
boolean pinned = PageHeader.isAcquired(absPageAddr);
@@ -2261,11 +2280,7 @@ public class PageMemoryImpl implements PageMemoryEx {
if (preparePageRemoval(fullPageId, absEvictAddr, saveDirtyPage)) {
loadedPages.remove(
fullPageId.groupId(),
- PageIdUtils.effectivePageId(fullPageId.pageId()),
- partTag(
- fullPageId.groupId(),
- PageIdUtils.partId(fullPageId.pageId())
- )
+ PageIdUtils.effectivePageId(fullPageId.pageId())
);
return addr;
@@ -2309,64 +2324,55 @@ public class PageMemoryImpl implements PageMemoryEx {
/**
* @param grpId Cache group ID.
* @param partId Partition ID.
- * @return Partition tag. Growing 1 based partition file version
+ * @return Partition generation. Growing, 1-based partition version. Changed
*/
- private int partTag(int grpId, int partId) {
+ private int partGeneration(int grpId, int partId) {
assert getReadHoldCount() > 0 || getWriteHoldCount() > 0;
- Integer tag = partTagMap.get(new T2<>(grpId, partId));
-
- if (tag == null)
- return 1;
+ Integer tag = partGenerationMap.get(new GroupPartitionId(grpId, partId));
- return tag;
+ return tag == null ? INIT_PART_GENERATION : tag;
}
/**
+ * Increments partition generation due to partition invalidation (e.g. partition was rebalanced to other node
+ * and evicted).
+ *
* @param grpId Cache group ID.
* @param partId Partition ID.
- * @return New partition tag.
+ * @return New partition generation.
*/
- private int incrementPartTag(int grpId, int partId) {
+ private int incrementPartGeneration(int grpId, int partId) {
assert getWriteHoldCount() > 0;
- T2<Integer, Integer> t = new T2<>(grpId, partId);
+ GroupPartitionId grpPart = new GroupPartitionId(grpId, partId);
- Integer tag = partTagMap.get(t);
+ Integer gen = partGenerationMap.get(grpPart);
- if (tag == null) {
- partTagMap.put(t, 2);
+ if (gen == null)
+ gen = INIT_PART_GENERATION;
- return 2;
- }
- else if (tag == Integer.MAX_VALUE) {
+ if (gen == Integer.MAX_VALUE) {
U.warn(log, "Partition tag overflow [grpId=" + grpId + ", partId=" + partId + "]");
- partTagMap.put(t, 0);
+ partGenerationMap.put(grpPart, 0);
return 0;
}
else {
- partTagMap.put(t, tag + 1);
+ partGenerationMap.put(grpPart, gen + 1);
- return tag + 1;
+ return gen + 1;
}
}
/**
* @param grpId Cache group id.
*/
- private void resetPartTags(int grpId) {
+ private void resetGroupPartitionsGeneration(int grpId) {
assert getWriteHoldCount() > 0;
- Iterator<T2<Integer, Integer>> iter = partTagMap.keySet().iterator();
-
- while (iter.hasNext()) {
- T2<Integer, Integer> t = iter.next();
-
- if (t.get1() == grpId)
- iter.remove();
- }
+ partGenerationMap.keySet().removeIf(grpPart -> grpPart.getGroupId() == grpId);
}
}
@@ -2552,14 +2558,19 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/**
+ * Sets pointer to checkpoint buffer.
+ *
* @param absPtr Page absolute pointer.
- * @param tmpRelPtr Temp buffer relative pointer.
+ * @param tmpRelPtr Temp buffer relative pointer or {@link #INVALID_REL_PTR} if page is not copied to checkpoint
+ * buffer.
*/
private static void tempBufferPointer(long absPtr, long tmpRelPtr) {
GridUnsafe.putLong(absPtr + PAGE_TMP_BUF_OFFSET, tmpRelPtr);
}
/**
+ * Gets pointer to checkpoint buffer or {@link #INVALID_REL_PTR} if page is not copied to checkpoint buffer.
+ *
* @param absPtr Page absolute pointer.
* @return Temp buffer relative pointer.
*/
@@ -2637,8 +2648,8 @@ public class PageMemoryImpl implements PageMemoryEx {
/** */
private Segment seg;
- /** */
- private GridPredicate3<Integer, Long, Integer> clearPred;
+ /** Clear element filter for (cache group ID, page ID). */
+ LoadedPagesMap.KeyPredicate clearPred;
/** */
private CountDownFuture doneFut;
@@ -2651,12 +2662,12 @@ public class PageMemoryImpl implements PageMemoryEx {
/**
* @param seg Segment.
- * @param clearPred Clear predicate.
+ * @param clearPred Clear predicate for (cache group ID, page ID).
* @param doneFut Completion future.
*/
private ClearSegmentRunnable(
Segment seg,
- GridPredicate3<Integer, Long, Integer> clearPred,
+ LoadedPagesMap.KeyPredicate clearPred,
boolean rmvDirty,
CountDownFuture doneFut,
int pageSize
@@ -2682,14 +2693,11 @@ public class PageMemoryImpl implements PageMemoryEx {
seg.writeLock().lock();
try {
- while (base < boundary) {
- long ptr = seg.loadedPages.clearAt(base, clearPred, INVALID_REL_PTR);
+ GridLongList list = seg.loadedPages.removeIf(base, boundary, clearPred);
- if (ptr != INVALID_REL_PTR)
- ptrs.add(ptr);
+ ptrs.addAll(list);
- base++;
- }
+ base = boundary;
}
finally {
seg.writeLock().unlock();
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f8ec79d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/ReplaceCandidate.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/ReplaceCandidate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/ReplaceCandidate.java
new file mode 100644
index 0000000..d760f07
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/ReplaceCandidate.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Replacement removal candidate. Class represents some page from loaded pages table. Usually candidate is found during
+ * random {@link LoadedPagesMap} touch.
+ */
+public class ReplaceCandidate {
+ /** Partition generation saved in map, too old value means page may be safely cleared. */
+ private int gen;
+
+ /** */
+ @GridToStringExclude
+ private long relPtr;
+
+ /** */
+ @GridToStringInclude
+ private FullPageId fullId;
+
+ /**
+ * @param gen Partition generation.
+ * @param relPtr Relative pointer to page.
+ * @param fullId Full page ID.
+ */
+ public ReplaceCandidate(int gen, long relPtr, FullPageId fullId) {
+ this.gen = gen;
+ this.relPtr = relPtr;
+ this.fullId = fullId;
+ }
+
+ /**
+ * @return Partition generation saved in map, too old value means page may be safely cleared.
+ */
+ public int generation() {
+ return gen;
+ }
+
+ /**
+ * @return Relative pointer to page.
+ */
+ public long relativePointer() {
+ return relPtr;
+ }
+
+ /**
+ * @return Index.
+ */
+ public FullPageId fullId() {
+ return fullId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ReplaceCandidate.class, this, "relPtr", U.hexLong(relPtr));
+ }
+}
[13/50] [abbrv] ignite git commit: IGNITE-7876: Adopt SVM Linear
Binary Classification Model and Trainer to the new Partitioned Dataset
Posted by sb...@apache.org.
IGNITE-7876: Adopt SVM Linear Binary Classification Model and Trainer to the new Partitioned Dataset
This closes #3607
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/318ffe50
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/318ffe50
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/318ffe50
Branch: refs/heads/ignite-zk
Commit: 318ffe50fc0edb1d31f05044be8367bd578b6a88
Parents: 29d56ae
Author: zaleslaw <za...@gmail.com>
Authored: Mon Mar 12 15:28:11 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Mon Mar 12 15:28:11 2018 +0300
----------------------------------------------------------------------
.../ml/svm/SVMBinaryClassificationExample.java | 131 --
.../ml/svm/SVMMultiClassificationExample.java | 130 --
.../src/main/resources/datasets/titanic.txt | 1309 ------------------
.../apache/ignite/ml/structures/Dataset.java | 10 +
.../ignite/ml/structures/LabeledDataset.java | 16 +-
.../svm/SVMLinearBinaryClassificationModel.java | 4 +-
.../SVMLinearBinaryClassificationTrainer.java | 87 +-
.../SVMLinearMultiClassClassificationModel.java | 89 --
...VMLinearMultiClassClassificationTrainer.java | 160 ---
.../ignite/ml/svm/SVMPartitionContext.java | 28 +
.../ml/svm/SVMPartitionDataBuilderOnHeap.java | 86 ++
.../org/apache/ignite/ml/LocalModelsTest.java | 28 -
.../org/apache/ignite/ml/svm/BaseSVMTest.java | 58 -
.../ignite/ml/svm/SVMBinaryTrainerTest.java | 74 +
.../org/apache/ignite/ml/svm/SVMModelTest.java | 15 -
.../org/apache/ignite/ml/svm/SVMTestSuite.java | 13 +-
...inearSVMBinaryClassificationTrainerTest.java | 35 -
...inearSVMBinaryClassificationTrainerTest.java | 141 --
...inearSVMBinaryClassificationTrainerTest.java | 38 -
...rSVMMultiClassClassificationTrainerTest.java | 35 -
...rSVMMultiClassClassificationTrainerTest.java | 76 -
...rSVMMultiClassClassificationTrainerTest.java | 38 -
22 files changed, 272 insertions(+), 2329 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMBinaryClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMBinaryClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMBinaryClassificationExample.java
deleted file mode 100644
index e256276..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMBinaryClassificationExample.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.svm;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Path;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.examples.ExampleNodeStartup;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.structures.LabeledDataset;
-import org.apache.ignite.ml.structures.LabeledDatasetTestTrainPair;
-import org.apache.ignite.ml.structures.preprocessing.LabeledDatasetLoader;
-import org.apache.ignite.ml.structures.preprocessing.LabellingMachine;
-import org.apache.ignite.ml.structures.preprocessing.Normalizer;
-import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
-import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationTrainer;
-import org.apache.ignite.thread.IgniteThread;
-
-/**
- * <p>
- * Example of using {@link org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel} with Titanic dataset.</p>
- * <p>
- * Note that in this example we cannot guarantee order in which nodes return results of intermediate
- * computations and therefore algorithm can return different results.</p>
- * <p>
- * Remote nodes should always be started with special configuration file which
- * enables P2P class loading: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will start node
- * with {@code examples/config/example-ignite.xml} configuration.</p>
- */
-public class SVMBinaryClassificationExample {
- /** Separator. */
- private static final String SEPARATOR = ",";
-
- /** Path to the Iris dataset. */
- private static final String TITANIC_DATASET = "examples/src/main/resources/datasets/titanic.txt";
-
- /**
- * Executes example.
- *
- * @param args Command line arguments, none required.
- */
- public static void main(String[] args) throws InterruptedException {
- System.out.println(">>> SVM Binary classification example started.");
- // Start ignite grid.
- try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- System.out.println(">>> Ignite grid started.");
-
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- SVMBinaryClassificationExample.class.getSimpleName(), () -> {
-
- try {
- // Prepare path to read
- File file = IgniteUtils.resolveIgnitePath(TITANIC_DATASET);
- if (file == null)
- throw new RuntimeException("Can't find file: " + TITANIC_DATASET);
-
- Path path = file.toPath();
-
- // Read dataset from file
- LabeledDataset dataset = LabeledDatasetLoader.loadFromTxtFile(path, SEPARATOR, true, false);
-
- // Normalize dataset
- Normalizer.normalizeWithMiniMax(dataset);
-
- // Random splitting of the given data as 70% train and 30% test datasets
- LabeledDatasetTestTrainPair split = new LabeledDatasetTestTrainPair(dataset, 0.3);
-
- System.out.println("\n>>> Amount of observations in train dataset " + split.train().rowSize());
- System.out.println("\n>>> Amount of observations in test dataset " + split.test().rowSize());
-
- LabeledDataset test = split.test();
- LabeledDataset train = split.train();
-
- System.out.println("\n>>> Create new linear binary SVM trainer object.");
- Trainer<SVMLinearBinaryClassificationModel, LabeledDataset> trainer = new SVMLinearBinaryClassificationTrainer();
-
- System.out.println("\n>>> Perform the training to get the model.");
- SVMLinearBinaryClassificationModel mdl = trainer.train(train);
-
- System.out.println("\n>>> SVM classification model: " + mdl);
-
- // Clone labels
- final double[] labels = test.labels();
-
- // Save predicted classes to test dataset
- LabellingMachine.assignLabels(test, mdl);
-
- // Calculate amount of errors on test dataset
- int amountOfErrors = 0;
- for (int i = 0; i < test.rowSize(); i++) {
- if (test.label(i) != labels[i])
- amountOfErrors++;
- }
-
- System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
- System.out.println("\n>>> Prediction percentage " + (1 - amountOfErrors / (double) test.rowSize()));
-
- } catch (IOException e) {
- e.printStackTrace();
- System.out.println("\n>>> Unexpected exception, check resources: " + e);
- } finally {
- System.out.println("\n>>> SVM binary classification example completed.");
- }
-
- });
-
- igniteThread.start();
- igniteThread.join();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMMultiClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMMultiClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMMultiClassificationExample.java
deleted file mode 100644
index 7aee5d2..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/svm/SVMMultiClassificationExample.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.svm;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Path;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.examples.ExampleNodeStartup;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.structures.LabeledDataset;
-import org.apache.ignite.ml.structures.LabeledDatasetTestTrainPair;
-import org.apache.ignite.ml.structures.preprocessing.LabeledDatasetLoader;
-import org.apache.ignite.ml.structures.preprocessing.LabellingMachine;
-import org.apache.ignite.ml.structures.preprocessing.Normalizer;
-import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationModel;
-import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationTrainer;
-import org.apache.ignite.thread.IgniteThread;
-
-/**
- * <p>
- * Example of using {@link org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationModel} with Iris dataset.</p>
- * <p>
- * Note that in this example we cannot guarantee order in which nodes return results of intermediate
- * computations and therefore algorithm can return different results.</p>
- * <p>
- * Remote nodes should always be started with special configuration file which
- * enables P2P class loading: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will start node
- * with {@code examples/config/example-ignite.xml} configuration.</p>
- */
-public class SVMMultiClassificationExample {
- /** Separator. */
- private static final String SEPARATOR = "\t";
-
- /** Path to the Iris dataset. */
- private static final String IRIS_DATASET = "examples/src/main/resources/datasets/iris.txt";
-
- /**
- * Executes example.
- *
- * @param args Command line arguments, none required.
- */
- public static void main(String[] args) throws InterruptedException {
- System.out.println(">>> SVM Multi-Class classification example started.");
- // Start ignite grid.
- try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- System.out.println(">>> Ignite grid started.");
-
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- SVMMultiClassificationExample.class.getSimpleName(), () -> {
-
- try {
- // Prepare path to read
- File file = IgniteUtils.resolveIgnitePath(IRIS_DATASET);
- if (file == null)
- throw new RuntimeException("Can't find file: " + IRIS_DATASET);
-
- Path path = file.toPath();
-
- // Read dataset from file
- LabeledDataset dataset = LabeledDatasetLoader.loadFromTxtFile(path, SEPARATOR, true, false);
-
- // Normalize dataset
- Normalizer.normalizeWithMiniMax(dataset);
-
- // Random splitting of the given data as 70% train and 30% test datasets
- LabeledDatasetTestTrainPair split = new LabeledDatasetTestTrainPair(dataset, 0.3);
-
- System.out.println("\n>>> Amount of observations in train dataset " + split.train().rowSize());
- System.out.println("\n>>> Amount of observations in test dataset " + split.test().rowSize());
-
- LabeledDataset test = split.test();
- LabeledDataset train = split.train();
-
- System.out.println("\n>>> Create new linear multi-class SVM trainer object.");
- Trainer<SVMLinearMultiClassClassificationModel, LabeledDataset> trainer = new SVMLinearMultiClassClassificationTrainer();
-
- System.out.println("\n>>> Perform the training to get the model.");
- SVMLinearMultiClassClassificationModel mdl = trainer.train(train);
-
- System.out.println("\n>>> SVM classification model: " + mdl);
-
- // Clone labels
- final double[] labels = test.labels();
-
- // Save predicted classes to test dataset
- LabellingMachine.assignLabels(test, mdl);
-
- // Calculate amount of errors on test dataset
- int amountOfErrors = 0;
- for (int i = 0; i < test.rowSize(); i++) {
- if (test.label(i) != labels[i])
- amountOfErrors++;
- }
-
- System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
- System.out.println("\n>>> Prediction percentage " + (1 - amountOfErrors / (double) test.rowSize()));
-
- } catch (IOException e) {
- e.printStackTrace();
- System.out.println("\n>>> Unexpected exception, check resources: " + e);
- } finally {
- System.out.println("\n>>> SVM Multi-Class classification example completed.");
- }
- });
-
- igniteThread.start();
- igniteThread.join();
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/examples/src/main/resources/datasets/titanic.txt
----------------------------------------------------------------------
diff --git a/examples/src/main/resources/datasets/titanic.txt b/examples/src/main/resources/datasets/titanic.txt
deleted file mode 100644
index fa2dc70..0000000
--- a/examples/src/main/resources/datasets/titanic.txt
+++ /dev/null
@@ -1,1309 +0,0 @@
-1,1,2,29,0,0,211.3375
-1,1,1,0.9167,1,2,151.55
--1,1,2,2,1,2,151.55
--1,1,1,30,1,2,151.55
--1,1,2,25,1,2,151.55
-1,1,1,48,0,0,26.55
-1,1,2,63,1,0,77.9583
--1,1,1,39,0,0,0
-1,1,2,53,2,0,51.4792
--1,1,1,71,0,0,49.5042
--1,1,1,47,1,0,227.525
-1,1,2,18,1,0,227.525
-1,1,2,24,0,0,69.3
-1,1,2,26,0,0,78.85
-1,1,1,80,0,0,30
--1,1,1,,0,0,25.925
--1,1,1,24,0,1,247.5208
-1,1,2,50,0,1,247.5208
-1,1,2,32,0,0,76.2917
--1,1,1,36,0,0,75.2417
-1,1,1,37,1,1,52.5542
-1,1,2,47,1,1,52.5542
-1,1,1,26,0,0,30
-1,1,2,42,0,0,227.525
-1,1,2,29,0,0,221.7792
--1,1,1,25,0,0,26
-1,1,1,25,1,0,91.0792
-1,1,2,19,1,0,91.0792
-1,1,2,35,0,0,135.6333
-1,1,1,28,0,0,26.55
--1,1,1,45,0,0,35.5
-1,1,1,40,0,0,31
-1,1,2,30,0,0,164.8667
-1,1,2,58,0,0,26.55
--1,1,1,42,0,0,26.55
-1,1,2,45,0,0,262.375
-1,1,2,22,0,1,55
-1,1,1,,0,0,26.55
--1,1,1,41,0,0,30.5
--1,1,1,48,0,0,50.4958
--1,1,1,,0,0,39.6
-1,1,2,44,0,0,27.7208
-1,1,2,59,2,0,51.4792
-1,1,2,60,0,0,76.2917
-1,1,2,41,0,0,134.5
--1,1,1,45,0,0,26.55
--1,1,1,,0,0,31
-1,1,1,42,0,0,26.2875
-1,1,2,53,0,0,27.4458
-1,1,1,36,0,1,512.3292
-1,1,2,58,0,1,512.3292
--1,1,1,33,0,0,5
--1,1,1,28,0,0,47.1
--1,1,1,17,0,0,47.1
-1,1,1,11,1,2,120
-1,1,2,14,1,2,120
-1,1,1,36,1,2,120
-1,1,2,36,1,2,120
--1,1,1,49,0,0,26
-1,1,2,,0,0,27.7208
--1,1,1,36,1,0,78.85
-1,1,2,76,1,0,78.85
--1,1,1,46,1,0,61.175
-1,1,2,47,1,0,61.175
-1,1,1,27,1,0,53.1
-1,1,2,33,1,0,53.1
-1,1,2,36,0,0,262.375
-1,1,2,30,0,0,86.5
-1,1,1,45,0,0,29.7
-1,1,2,,0,1,55
--1,1,1,,0,0,0
--1,1,1,27,1,0,136.7792
-1,1,2,26,1,0,136.7792
-1,1,2,22,0,0,151.55
--1,1,1,,0,0,52
--1,1,1,47,0,0,25.5875
-1,1,2,39,1,1,83.1583
--1,1,1,37,1,1,83.1583
-1,1,2,64,0,2,83.1583
-1,1,2,55,2,0,25.7
--1,1,1,,0,0,26.55
--1,1,1,70,1,1,71
-1,1,2,36,0,2,71
-1,1,2,64,1,1,26.55
--1,1,1,39,1,0,71.2833
-1,1,2,38,1,0,71.2833
-1,1,1,51,0,0,26.55
-1,1,1,27,0,0,30.5
-1,1,2,33,0,0,151.55
--1,1,1,31,1,0,52
-1,1,2,27,1,2,52
-1,1,1,31,1,0,57
-1,1,2,17,1,0,57
-1,1,1,53,1,1,81.8583
-1,1,1,4,0,2,81.8583
-1,1,2,54,1,1,81.8583
--1,1,1,50,1,0,106.425
-1,1,2,27,1,1,247.5208
-1,1,2,48,1,0,106.425
-1,1,2,48,1,0,39.6
-1,1,1,49,1,0,56.9292
--1,1,1,39,0,0,29.7
-1,1,2,23,0,1,83.1583
-1,1,2,38,0,0,227.525
-1,1,2,54,1,0,78.2667
--1,1,2,36,0,0,31.6792
--1,1,1,,0,0,221.7792
-1,1,2,,0,0,31.6833
-1,1,2,,0,0,110.8833
-1,1,1,36,0,0,26.3875
--1,1,1,30,0,0,27.75
-1,1,2,24,3,2,263
-1,1,2,28,3,2,263
-1,1,2,23,3,2,263
--1,1,1,19,3,2,263
--1,1,1,64,1,4,263
-1,1,2,60,1,4,263
-1,1,2,30,0,0,56.9292
--1,1,1,,0,0,26.55
-1,1,1,50,2,0,133.65
-1,1,1,43,1,0,27.7208
-1,1,2,,1,0,133.65
-1,1,2,22,0,2,49.5
-1,1,1,60,1,1,79.2
-1,1,2,48,1,1,79.2
--1,1,1,,0,0,0
--1,1,1,37,1,0,53.1
-1,1,2,35,1,0,53.1
--1,1,1,47,0,0,38.5
-1,1,2,35,0,0,211.5
-1,1,2,22,0,1,59.4
-1,1,2,45,0,1,59.4
--1,1,1,24,0,0,79.2
-1,1,1,49,1,0,89.1042
-1,1,2,,1,0,89.1042
--1,1,1,71,0,0,34.6542
-1,1,1,53,0,0,28.5
-1,1,2,19,0,0,30
--1,1,1,38,0,1,153.4625
-1,1,2,58,0,1,153.4625
-1,1,1,23,0,1,63.3583
-1,1,2,45,0,1,63.3583
--1,1,1,46,0,0,79.2
-1,1,1,25,1,0,55.4417
-1,1,2,25,1,0,55.4417
-1,1,1,48,1,0,76.7292
-1,1,2,49,1,0,76.7292
--1,1,1,,0,0,42.4
--1,1,1,45,1,0,83.475
-1,1,2,35,1,0,83.475
--1,1,1,40,0,0,0
-1,1,1,27,0,0,76.7292
-1,1,1,,0,0,30
-1,1,2,24,0,0,83.1583
--1,1,1,55,1,1,93.5
-1,1,2,52,1,1,93.5
--1,1,1,42,0,0,42.5
--1,1,1,,0,0,51.8625
--1,1,1,55,0,0,50
-1,1,2,16,0,1,57.9792
-1,1,2,44,0,1,57.9792
-1,1,2,51,1,0,77.9583
--1,1,1,42,1,0,52
-1,1,2,35,1,0,52
-1,1,1,35,0,0,26.55
-1,1,1,38,1,0,90
--1,1,1,,0,0,30.6958
-1,1,2,35,1,0,90
-1,1,2,38,0,0,80
--1,1,2,50,0,0,28.7125
-1,1,1,49,0,0,0
--1,1,1,46,0,0,26
--1,1,1,50,0,0,26
--1,1,1,32.5,0,0,211.5
--1,1,1,58,0,0,29.7
--1,1,1,41,1,0,51.8625
-1,1,2,,1,0,51.8625
-1,1,1,42,1,0,52.5542
-1,1,2,45,1,0,52.5542
--1,1,1,,0,0,26.55
-1,1,2,39,0,0,211.3375
-1,1,2,49,0,0,25.9292
-1,1,2,30,0,0,106.425
-1,1,1,35,0,0,512.3292
--1,1,1,,0,0,27.7208
--1,1,1,42,0,0,26.55
-1,1,2,55,0,0,27.7208
-1,1,2,16,0,1,39.4
-1,1,2,51,0,1,39.4
--1,1,1,29,0,0,30
-1,1,2,21,0,0,77.9583
--1,1,1,30,0,0,45.5
-1,1,2,58,0,0,146.5208
-1,1,2,15,0,1,211.3375
--1,1,1,30,0,0,26
-1,1,2,16,0,0,86.5
-1,1,1,,0,0,29.7
--1,1,1,19,1,0,53.1
-1,1,2,18,1,0,53.1
-1,1,2,24,0,0,49.5042
--1,1,1,46,0,0,75.2417
--1,1,1,54,0,0,51.8625
-1,1,1,36,0,0,26.2875
--1,1,1,28,1,0,82.1708
-1,1,2,,1,0,82.1708
--1,1,1,65,0,0,26.55
--1,1,1,44,2,0,90
-1,1,2,33,1,0,90
-1,1,2,37,1,0,90
-1,1,1,30,1,0,57.75
--1,1,1,55,0,0,30.5
--1,1,1,47,0,0,42.4
--1,1,1,37,0,1,29.7
-1,1,2,31,1,0,113.275
-1,1,2,23,1,0,113.275
--1,1,1,58,0,2,113.275
-1,1,2,19,0,2,26.2833
--1,1,1,64,0,0,26
-1,1,2,39,0,0,108.9
-1,1,1,,0,0,25.7417
-1,1,2,22,0,1,61.9792
--1,1,1,65,0,1,61.9792
--1,1,1,28.5,0,0,27.7208
--1,1,1,,0,0,0
--1,1,1,45.5,0,0,28.5
--1,1,1,23,0,0,93.5
--1,1,1,29,1,0,66.6
-1,1,2,22,1,0,66.6
--1,1,1,18,1,0,108.9
-1,1,2,17,1,0,108.9
-1,1,2,30,0,0,93.5
-1,1,1,52,0,0,30.5
--1,1,1,47,0,0,52
-1,1,2,56,0,1,83.1583
--1,1,1,38,0,0,0
-1,1,1,,0,0,39.6
--1,1,1,22,0,0,135.6333
--1,1,1,,0,0,227.525
-1,1,2,43,0,1,211.3375
--1,1,1,31,0,0,50.4958
-1,1,1,45,0,0,26.55
--1,1,1,,0,0,50
-1,1,2,33,0,0,27.7208
--1,1,1,46,0,0,79.2
--1,1,1,36,0,0,40.125
-1,1,2,33,0,0,86.5
--1,1,1,55,1,0,59.4
-1,1,2,54,1,0,59.4
--1,1,1,33,0,0,26.55
-1,1,1,13,2,2,262.375
-1,1,2,18,2,2,262.375
-1,1,2,21,2,2,262.375
--1,1,1,61,1,3,262.375
-1,1,2,48,1,3,262.375
-1,1,1,,0,0,30.5
-1,1,2,24,0,0,69.3
-1,1,1,,0,0,26
-1,1,2,35,1,0,57.75
-1,1,2,30,0,0,31
-1,1,1,34,0,0,26.55
-1,1,2,40,0,0,153.4625
-1,1,1,35,0,0,26.2875
--1,1,1,50,1,0,55.9
-1,1,2,39,1,0,55.9
-1,1,1,56,0,0,35.5
-1,1,1,28,0,0,35.5
--1,1,1,56,0,0,26.55
--1,1,1,56,0,0,30.6958
--1,1,1,24,1,0,60
--1,1,1,,0,0,26
-1,1,2,18,1,0,60
-1,1,1,24,1,0,82.2667
-1,1,2,23,1,0,82.2667
-1,1,1,6,0,2,134.5
-1,1,1,45,1,1,134.5
-1,1,2,40,1,1,134.5
--1,1,1,57,1,0,146.5208
-1,1,2,,1,0,146.5208
-1,1,1,32,0,0,30.5
--1,1,1,62,0,0,26.55
-1,1,1,54,1,0,55.4417
-1,1,2,43,1,0,55.4417
-1,1,2,52,1,0,78.2667
--1,1,1,,0,0,27.7208
-1,1,2,62,0,0,80
--1,1,1,67,1,0,221.7792
--1,1,2,63,1,0,221.7792
--1,1,1,61,0,0,32.3208
-1,1,2,48,0,0,25.9292
-1,1,2,18,0,2,79.65
--1,1,1,52,1,1,79.65
-1,1,2,39,1,1,79.65
-1,1,1,48,1,0,52
-1,1,2,,1,0,52
--1,1,1,49,1,1,110.8833
-1,1,1,17,0,2,110.8833
-1,1,2,39,1,1,110.8833
-1,1,2,,0,0,79.2
-1,1,1,31,0,0,28.5375
--1,1,1,40,0,0,27.7208
--1,1,1,61,0,0,33.5
--1,1,1,47,0,0,34.0208
-1,1,2,35,0,0,512.3292
--1,1,1,64,1,0,75.25
-1,1,2,60,1,0,75.25
--1,1,1,60,0,0,26.55
--1,1,1,54,0,1,77.2875
--1,1,1,21,0,1,77.2875
-1,1,2,55,0,0,135.6333
-1,1,2,31,0,2,164.8667
--1,1,1,57,1,1,164.8667
-1,1,2,45,1,1,164.8667
--1,1,1,50,1,1,211.5
--1,1,1,27,0,2,211.5
-1,1,2,50,1,1,211.5
-1,1,2,21,0,0,26.55
--1,1,1,51,0,1,61.3792
-1,1,1,21,0,1,61.3792
--1,1,1,,0,0,35
-1,1,2,31,0,0,134.5
-1,1,1,,0,0,35.5
--1,1,1,62,0,0,26.55
-1,1,2,36,0,0,135.6333
--1,2,1,30,1,0,24
-1,2,2,28,1,0,24
--1,2,1,30,0,0,13
--1,2,1,18,0,0,11.5
--1,2,1,25,0,0,10.5
--1,2,1,34,1,0,26
-1,2,2,36,1,0,26
--1,2,1,57,0,0,13
--1,2,1,18,0,0,11.5
--1,2,1,23,0,0,10.5
-1,2,2,36,0,0,13
--1,2,1,28,0,0,10.5
--1,2,1,51,0,0,12.525
-1,2,1,32,1,0,26
-1,2,2,19,1,0,26
--1,2,1,28,0,0,26
-1,2,1,1,2,1,39
-1,2,2,4,2,1,39
-1,2,2,12,2,1,39
-1,2,2,36,0,3,39
-1,2,1,34,0,0,13
-1,2,2,19,0,0,13
--1,2,1,23,0,0,13
--1,2,1,26,0,0,13
--1,2,1,42,0,0,13
--1,2,1,27,0,0,13
-1,2,2,24,0,0,13
-1,2,2,15,0,2,39
--1,2,1,60,1,1,39
-1,2,2,40,1,1,39
-1,2,2,20,1,0,26
--1,2,1,25,1,0,26
-1,2,2,36,0,0,13
--1,2,1,25,0,0,13
--1,2,1,42,0,0,13
-1,2,2,42,0,0,13
-1,2,1,0.8333,0,2,29
-1,2,1,26,1,1,29
-1,2,2,22,1,1,29
-1,2,2,35,0,0,21
--1,2,1,,0,0,0
--1,2,1,19,0,0,13
--1,2,2,44,1,0,26
--1,2,1,54,1,0,26
--1,2,1,52,0,0,13.5
--1,2,1,37,1,0,26
--1,2,2,29,1,0,26
-1,2,2,25,1,1,30
-1,2,2,45,0,2,30
--1,2,1,29,1,0,26
-1,2,2,28,1,0,26
--1,2,1,29,0,0,10.5
--1,2,1,28,0,0,13
-1,2,1,24,0,0,10.5
-1,2,2,8,0,2,26.25
--1,2,1,31,1,1,26.25
-1,2,2,31,1,1,26.25
-1,2,2,22,0,0,10.5
--1,2,2,30,0,0,13
--1,2,2,,0,0,21
--1,2,1,21,0,0,11.5
--1,2,1,,0,0,0
-1,2,1,8,1,1,36.75
--1,2,1,18,0,0,73.5
-1,2,2,48,0,2,36.75
-1,2,2,28,0,0,13
--1,2,1,32,0,0,13
--1,2,1,17,0,0,73.5
--1,2,1,29,1,0,27.7208
-1,2,2,24,1,0,27.7208
--1,2,1,25,0,0,31.5
--1,2,1,18,0,0,73.5
-1,2,2,18,0,1,23
-1,2,2,34,0,1,23
--1,2,1,54,0,0,26
-1,2,1,8,0,2,32.5
--1,2,1,42,1,1,32.5
-1,2,2,34,1,1,32.5
-1,2,2,27,1,0,13.8583
-1,2,2,30,1,0,13.8583
--1,2,1,23,0,0,13
--1,2,1,21,0,0,13
--1,2,1,18,0,0,13
--1,2,1,40,1,0,26
-1,2,2,29,1,0,26
--1,2,1,18,0,0,10.5
--1,2,1,36,0,0,13
--1,2,1,,0,0,0
--1,2,2,38,0,0,13
--1,2,1,35,0,0,26
--1,2,1,38,1,0,21
--1,2,1,34,1,0,21
-1,2,2,34,0,0,13
--1,2,1,16,0,0,26
--1,2,1,26,0,0,10.5
--1,2,1,47,0,0,10.5
--1,2,1,21,1,0,11.5
--1,2,1,21,1,0,11.5
--1,2,1,24,0,0,13.5
--1,2,1,24,0,0,13
--1,2,1,34,0,0,13
--1,2,1,30,0,0,13
--1,2,1,52,0,0,13
--1,2,1,30,0,0,13
-1,2,1,0.6667,1,1,14.5
-1,2,2,24,0,2,14.5
--1,2,1,44,0,0,13
-1,2,2,6,0,1,33
--1,2,1,28,0,1,33
-1,2,1,62,0,0,10.5
--1,2,1,30,0,0,10.5
-1,2,2,7,0,2,26.25
--1,2,1,43,1,1,26.25
-1,2,2,45,1,1,26.25
-1,2,2,24,1,2,65
-1,2,2,24,1,2,65
--1,2,1,49,1,2,65
-1,2,2,48,1,2,65
-1,2,2,55,0,0,16
--1,2,1,24,2,0,73.5
--1,2,1,32,2,0,73.5
--1,2,1,21,2,0,73.5
--1,2,2,18,1,1,13
-1,2,2,20,2,1,23
--1,2,1,23,2,1,11.5
--1,2,1,36,0,0,13
-1,2,2,54,1,3,23
--1,2,1,50,0,0,13
--1,2,1,44,1,0,26
-1,2,2,29,1,0,26
--1,2,1,21,0,0,73.5
-1,2,1,42,0,0,13
--1,2,1,63,1,0,26
--1,2,2,60,1,0,26
--1,2,1,33,0,0,12.275
-1,2,2,17,0,0,10.5
--1,2,1,42,1,0,27
-1,2,2,24,2,1,27
--1,2,1,47,0,0,15
--1,2,1,24,2,0,31.5
--1,2,1,22,2,0,31.5
--1,2,1,32,0,0,10.5
-1,2,2,23,0,0,13.7917
--1,2,1,34,1,0,26
-1,2,2,24,1,0,26
--1,2,2,22,0,0,21
-1,2,2,,0,0,12.35
--1,2,1,35,0,0,12.35
-1,2,2,45,0,0,13.5
--1,2,1,57,0,0,12.35
--1,2,1,,0,0,0
--1,2,1,31,0,0,10.5
--1,2,2,26,1,1,26
--1,2,1,30,1,1,26
--1,2,1,,0,0,10.7083
-1,2,2,1,1,2,41.5792
-1,2,2,3,1,2,41.5792
--1,2,1,25,1,2,41.5792
-1,2,2,22,1,2,41.5792
-1,2,2,17,0,0,12
-1,2,2,,0,0,33
-1,2,2,34,0,0,10.5
--1,2,1,36,0,0,12.875
--1,2,1,24,0,0,10.5
--1,2,1,61,0,0,12.35
--1,2,1,50,1,0,26
-1,2,2,42,1,0,26
--1,2,2,57,0,0,10.5
--1,2,1,,0,0,15.0458
-1,2,1,1,0,2,37.0042
--1,2,1,31,1,1,37.0042
-1,2,2,24,1,1,37.0042
--1,2,1,,0,0,15.5792
--1,2,1,30,0,0,13
--1,2,1,40,0,0,16
--1,2,1,32,0,0,13.5
--1,2,1,30,0,0,13
--1,2,1,46,0,0,26
-1,2,2,13,0,1,19.5
-1,2,2,41,0,1,19.5
-1,2,1,19,0,0,10.5
--1,2,1,39,0,0,13
--1,2,1,48,0,0,13
--1,2,1,70,0,0,10.5
--1,2,1,27,0,0,13
--1,2,1,54,0,0,14
--1,2,1,39,0,0,26
--1,2,1,16,0,0,10.5
--1,2,1,62,0,0,9.6875
--1,2,1,32.5,1,0,30.0708
-1,2,2,14,1,0,30.0708
-1,2,1,2,1,1,26
-1,2,1,3,1,1,26
--1,2,1,36.5,0,2,26
--1,2,1,26,0,0,13
--1,2,1,19,1,1,36.75
--1,2,1,28,0,0,13.5
-1,2,1,20,0,0,13.8625
-1,2,2,29,0,0,10.5
--1,2,1,39,0,0,13
-1,2,1,22,0,0,10.5
-1,2,1,,0,0,13.8625
--1,2,1,23,0,0,10.5
-1,2,1,29,0,0,13.8583
--1,2,1,28,0,0,10.5
--1,2,1,,0,0,0
-1,2,2,50,0,1,26
--1,2,1,19,0,0,10.5
--1,2,1,,0,0,15.05
--1,2,1,41,0,0,13
-1,2,2,21,0,1,21
-1,2,2,19,0,0,26
--1,2,1,43,0,1,21
-1,2,2,32,0,0,13
--1,2,1,34,0,0,13
-1,2,1,30,0,0,12.7375
--1,2,1,27,0,0,15.0333
-1,2,2,2,1,1,26
-1,2,2,8,1,1,26
-1,2,2,33,0,2,26
--1,2,1,36,0,0,10.5
--1,2,1,34,1,0,21
-1,2,2,30,3,0,21
-1,2,2,28,0,0,13
--1,2,1,23,0,0,15.0458
-1,2,1,0.8333,1,1,18.75
-1,2,1,3,1,1,18.75
-1,2,2,24,2,3,18.75
-1,2,2,50,0,0,10.5
--1,2,1,19,0,0,10.5
-1,2,2,21,0,0,10.5
--1,2,1,26,0,0,13
--1,2,1,25,0,0,13
--1,2,1,27,0,0,26
-1,2,2,25,0,1,26
-1,2,2,18,0,2,13
-1,2,2,20,0,0,36.75
-1,2,2,30,0,0,13
--1,2,1,59,0,0,13.5
-1,2,2,30,0,0,12.35
--1,2,1,35,0,0,10.5
-1,2,2,40,0,0,13
--1,2,1,25,0,0,13
--1,2,1,41,0,0,15.0458
--1,2,1,25,0,0,10.5
--1,2,1,18.5,0,0,13
--1,2,1,14,0,0,65
-1,2,2,50,0,0,10.5
--1,2,1,23,0,0,13
-1,2,2,28,0,0,12.65
-1,2,2,27,0,0,10.5
--1,2,1,29,1,0,21
--1,2,2,27,1,0,21
--1,2,1,40,0,0,13
-1,2,2,31,0,0,21
--1,2,1,30,1,0,21
--1,2,1,23,1,0,10.5
-1,2,2,31,0,0,21
--1,2,1,,0,0,0
-1,2,2,12,0,0,15.75
-1,2,2,40,0,0,15.75
-1,2,2,32.5,0,0,13
--1,2,1,27,1,0,26
-1,2,2,29,1,0,26
-1,2,1,2,1,1,23
-1,2,2,4,1,1,23
-1,2,2,29,0,2,23
-1,2,2,0.9167,1,2,27.75
-1,2,2,5,1,2,27.75
--1,2,1,36,1,2,27.75
-1,2,2,33,1,2,27.75
--1,2,1,66,0,0,10.5
--1,2,1,,0,0,12.875
-1,2,1,31,0,0,13
-1,2,1,,0,0,13
-1,2,2,26,0,0,13.5
--1,2,2,24,0,0,13
--1,3,1,42,0,0,7.55
--1,3,1,13,0,2,20.25
--1,3,1,16,1,1,20.25
-1,3,2,35,1,1,20.25
-1,3,2,16,0,0,7.65
-1,3,1,25,0,0,7.65
-1,3,1,20,0,0,7.925
-1,3,2,18,0,0,7.2292
--1,3,1,30,0,0,7.25
--1,3,1,26,0,0,8.05
--1,3,2,40,1,0,9.475
-1,3,1,0.8333,0,1,9.35
-1,3,2,18,0,1,9.35
-1,3,1,26,0,0,18.7875
--1,3,1,26,0,0,7.8875
--1,3,1,20,0,0,7.925
--1,3,1,24,0,0,7.05
--1,3,1,25,0,0,7.05
--1,3,1,35,0,0,8.05
--1,3,1,18,0,0,8.3
--1,3,1,32,0,0,22.525
-1,3,2,19,1,0,7.8542
--1,3,1,4,4,2,31.275
--1,3,2,6,4,2,31.275
--1,3,2,2,4,2,31.275
-1,3,2,17,4,2,7.925
--1,3,2,38,4,2,7.775
--1,3,2,9,4,2,31.275
--1,3,2,11,4,2,31.275
--1,3,1,39,1,5,31.275
-1,3,1,27,0,0,7.7958
--1,3,1,26,0,0,7.775
--1,3,2,39,1,5,31.275
--1,3,1,20,0,0,7.8542
--1,3,1,26,0,0,7.8958
--1,3,1,25,1,0,17.8
--1,3,2,18,1,0,17.8
--1,3,1,24,0,0,7.775
--1,3,1,35,0,0,7.05
--1,3,1,5,4,2,31.3875
--1,3,1,9,4,2,31.3875
-1,3,1,3,4,2,31.3875
--1,3,1,13,4,2,31.3875
-1,3,2,5,4,2,31.3875
--1,3,1,40,1,5,31.3875
-1,3,1,23,0,0,7.7958
-1,3,2,38,1,5,31.3875
-1,3,2,45,0,0,7.225
--1,3,1,21,0,0,7.225
--1,3,1,23,0,0,7.05
--1,3,2,17,0,0,14.4583
--1,3,1,30,0,0,7.225
--1,3,1,23,0,0,7.8542
-1,3,2,13,0,0,7.2292
--1,3,1,20,0,0,7.225
--1,3,1,32,1,0,15.85
-1,3,2,33,3,0,15.85
-1,3,2,0.75,2,1,19.2583
-1,3,2,0.75,2,1,19.2583
-1,3,2,5,2,1,19.2583
-1,3,2,24,0,3,19.2583
-1,3,2,18,0,0,8.05
--1,3,1,40,0,0,7.225
--1,3,1,26,0,0,7.8958
-1,3,1,20,0,0,7.2292
--1,3,2,18,0,1,14.4542
--1,3,2,45,0,1,14.4542
--1,3,2,27,0,0,7.8792
--1,3,1,22,0,0,8.05
--1,3,1,19,0,0,8.05
--1,3,1,26,0,0,7.775
--1,3,1,22,0,0,9.35
--1,3,1,,0,0,7.2292
--1,3,1,20,0,0,4.0125
-1,3,1,32,0,0,56.4958
--1,3,1,21,0,0,7.775
--1,3,1,18,0,0,7.75
--1,3,1,26,0,0,7.8958
--1,3,1,6,1,1,15.2458
--1,3,2,9,1,1,15.2458
--1,3,1,,0,0,7.225
--1,3,2,,0,2,15.2458
--1,3,2,,0,2,7.75
--1,3,1,40,1,1,15.5
--1,3,2,32,1,1,15.5
--1,3,1,21,0,0,16.1
-1,3,2,22,0,0,7.725
--1,3,2,20,0,0,7.8542
--1,3,1,29,1,0,7.0458
--1,3,1,22,1,0,7.25
--1,3,1,22,0,0,7.7958
--1,3,1,35,0,0,8.05
--1,3,2,18.5,0,0,7.2833
-1,3,1,21,0,0,7.8208
--1,3,1,19,0,0,6.75
--1,3,2,18,0,0,7.8792
--1,3,2,21,0,0,8.6625
--1,3,2,30,0,0,8.6625
--1,3,1,18,0,0,8.6625
--1,3,1,38,0,0,8.6625
--1,3,1,17,0,0,8.6625
--1,3,1,17,0,0,8.6625
--1,3,2,21,0,0,7.75
--1,3,1,21,0,0,7.75
--1,3,1,21,0,0,8.05
--1,3,1,,1,0,14.4583
--1,3,2,,1,0,14.4583
--1,3,1,28,0,0,7.7958
--1,3,1,24,0,0,7.8542
-1,3,2,16,0,0,7.75
--1,3,2,37,0,0,7.75
--1,3,1,28,0,0,7.25
--1,3,1,24,0,0,8.05
--1,3,1,21,0,0,7.7333
-1,3,1,32,0,0,56.4958
--1,3,1,29,0,0,8.05
--1,3,1,26,1,0,14.4542
--1,3,1,18,1,0,14.4542
--1,3,1,20,0,0,7.05
-1,3,1,18,0,0,8.05
--1,3,1,24,0,0,7.25
--1,3,1,36,0,0,7.4958
--1,3,1,24,0,0,7.4958
--1,3,1,31,0,0,7.7333
--1,3,1,31,0,0,7.75
-1,3,2,22,0,0,7.75
--1,3,2,30,0,0,7.6292
--1,3,1,70.5,0,0,7.75
--1,3,1,43,0,0,8.05
--1,3,1,35,0,0,7.8958
--1,3,1,27,0,0,7.8958
--1,3,1,19,0,0,7.8958
--1,3,1,30,0,0,8.05
-1,3,1,9,1,1,15.9
-1,3,1,3,1,1,15.9
-1,3,2,36,0,2,15.9
--1,3,1,59,0,0,7.25
--1,3,1,19,0,0,8.1583
-1,3,2,17,0,1,16.1
--1,3,1,44,0,1,16.1
--1,3,1,17,0,0,8.6625
--1,3,1,22.5,0,0,7.225
-1,3,1,45,0,0,8.05
--1,3,2,22,0,0,10.5167
--1,3,1,19,0,0,10.1708
-1,3,2,30,0,0,6.95
-1,3,1,29,0,0,7.75
--1,3,1,0.3333,0,2,14.4
--1,3,1,34,1,1,14.4
--1,3,2,28,1,1,14.4
--1,3,1,27,0,0,7.8958
--1,3,1,25,0,0,7.8958
--1,3,1,24,2,0,24.15
--1,3,1,22,0,0,8.05
--1,3,1,21,2,0,24.15
--1,3,1,17,2,0,8.05
--1,3,1,,1,0,16.1
-1,3,2,,1,0,16.1
-1,3,1,36.5,1,0,17.4
-1,3,2,36,1,0,17.4
-1,3,1,30,0,0,9.5
--1,3,1,16,0,0,9.5
-1,3,1,1,1,2,20.575
-1,3,2,0.1667,1,2,20.575
--1,3,1,26,1,2,20.575
-1,3,2,33,1,2,20.575
--1,3,1,25,0,0,7.8958
--1,3,1,,0,0,7.8958
--1,3,1,,0,0,7.8958
--1,3,1,22,0,0,7.25
--1,3,1,36,0,0,7.25
-1,3,2,19,0,0,7.8792
--1,3,1,17,0,0,7.8958
--1,3,1,42,0,0,8.6625
--1,3,1,43,0,0,7.8958
--1,3,1,,0,0,7.2292
--1,3,1,32,0,0,7.75
-1,3,1,19,0,0,8.05
-1,3,2,30,0,0,12.475
--1,3,2,24,0,0,7.75
-1,3,2,23,0,0,8.05
--1,3,1,33,0,0,7.8958
--1,3,1,65,0,0,7.75
-1,3,1,24,0,0,7.55
--1,3,1,23,1,0,13.9
-1,3,2,22,1,0,13.9
--1,3,1,18,0,0,7.775
--1,3,1,16,0,0,7.775
--1,3,1,45,0,0,6.975
--1,3,1,,0,0,7.225
--1,3,1,39,0,2,7.2292
--1,3,1,17,1,1,7.2292
--1,3,1,15,1,1,7.2292
--1,3,1,47,0,0,7.25
-1,3,2,5,0,0,12.475
--1,3,1,,0,0,7.225
--1,3,1,40.5,0,0,15.1
--1,3,1,40.5,0,0,7.75
-1,3,1,,0,0,7.05
--1,3,1,18,0,0,7.7958
--1,3,2,,0,0,7.75
--1,3,1,,0,0,7.75
--1,3,1,,0,0,6.95
--1,3,1,26,0,0,7.8792
--1,3,1,,0,0,7.75
-1,3,1,,0,0,56.4958
--1,3,2,21,2,2,34.375
--1,3,2,9,2,2,34.375
--1,3,1,,0,0,8.05
--1,3,1,18,2,2,34.375
--1,3,1,16,1,3,34.375
--1,3,2,48,1,3,34.375
--1,3,1,,0,0,7.75
--1,3,1,,0,0,7.25
--1,3,1,25,0,0,7.7417
--1,3,1,,0,0,14.5
--1,3,1,,0,0,7.8958
--1,3,1,22,0,0,8.05
-1,3,2,16,0,0,7.7333
-1,3,2,,0,0,7.75
-1,3,1,9,0,2,20.525
--1,3,1,33,1,1,20.525
--1,3,1,41,0,0,7.85
-1,3,2,31,1,1,20.525
--1,3,1,38,0,0,7.05
--1,3,1,9,5,2,46.9
--1,3,1,1,5,2,46.9
--1,3,1,11,5,2,46.9
--1,3,2,10,5,2,46.9
--1,3,2,16,5,2,46.9
--1,3,1,14,5,2,46.9
--1,3,1,40,1,6,46.9
--1,3,2,43,1,6,46.9
--1,3,1,51,0,0,8.05
--1,3,1,32,0,0,8.3625
--1,3,1,,0,0,8.05
--1,3,1,20,0,0,9.8458
--1,3,1,37,2,0,7.925
--1,3,1,28,2,0,7.925
--1,3,1,19,0,0,7.775
--1,3,2,24,0,0,8.85
--1,3,2,17,0,0,7.7333
--1,3,1,,1,0,19.9667
--1,3,1,,1,0,19.9667
--1,3,1,28,1,0,15.85
-1,3,2,24,1,0,15.85
--1,3,1,20,0,0,9.5
--1,3,1,23.5,0,0,7.2292
--1,3,1,41,2,0,14.1083
--1,3,1,26,1,0,7.8542
--1,3,1,21,0,0,7.8542
-1,3,2,45,1,0,14.1083
--1,3,2,,0,0,7.55
--1,3,1,25,0,0,7.25
--1,3,1,,0,0,6.8583
--1,3,1,11,0,0,18.7875
-1,3,2,,0,0,7.75
-1,3,1,27,0,0,6.975
-1,3,1,,0,0,56.4958
--1,3,2,18,0,0,6.75
-1,3,2,26,0,0,7.925
--1,3,2,23,0,0,7.925
-1,3,2,22,0,0,8.9625
--1,3,1,28,0,0,7.8958
--1,3,2,28,0,0,7.775
--1,3,2,,0,0,7.75
-1,3,2,2,0,1,12.2875
-1,3,2,22,1,1,12.2875
--1,3,1,43,0,0,6.45
--1,3,1,28,0,0,22.525
-1,3,2,27,0,0,7.925
--1,3,1,,0,0,7.75
-1,3,2,,0,0,8.05
--1,3,1,42,0,0,7.65
-1,3,1,,0,0,7.8875
--1,3,1,30,0,0,7.2292
--1,3,1,,0,0,7.8958
--1,3,2,27,1,0,7.925
--1,3,2,25,1,0,7.925
--1,3,1,,0,0,7.8958
-1,3,1,29,0,0,7.8958
-1,3,1,21,0,0,7.7958
--1,3,1,,0,0,7.05
--1,3,1,20,0,0,7.8542
--1,3,1,48,0,0,7.8542
--1,3,1,17,1,0,7.0542
-1,3,2,,0,0,7.75
-1,3,1,,0,0,8.1125
--1,3,1,34,0,0,6.4958
-1,3,1,26,0,0,7.775
--1,3,1,22,0,0,7.7958
--1,3,1,33,0,0,8.6542
--1,3,1,31,0,0,7.775
--1,3,1,29,0,0,7.8542
-1,3,1,4,1,1,11.1333
-1,3,2,1,1,1,11.1333
--1,3,1,49,0,0,0
--1,3,1,33,0,0,7.775
--1,3,1,19,0,0,0
-1,3,2,27,0,2,11.1333
--1,3,1,,1,2,23.45
--1,3,2,,1,2,23.45
--1,3,1,,1,2,23.45
--1,3,2,,1,2,23.45
--1,3,1,23,0,0,7.8958
-1,3,1,32,0,0,7.8542
--1,3,1,27,0,0,7.8542
--1,3,2,20,1,0,9.825
--1,3,2,21,1,0,9.825
-1,3,1,32,0,0,7.925
--1,3,1,17,0,0,7.125
--1,3,1,21,0,0,8.4333
--1,3,1,30,0,0,7.8958
-1,3,1,21,0,0,7.7958
--1,3,1,33,0,0,7.8542
--1,3,1,22,0,0,7.5208
-1,3,2,4,0,1,13.4167
-1,3,1,39,0,1,13.4167
--1,3,1,,0,0,7.2292
--1,3,1,18.5,0,0,7.2292
--1,3,1,,0,0,7.75
--1,3,1,,0,0,7.25
-1,3,2,,0,0,7.75
-1,3,2,,0,0,7.75
--1,3,1,34.5,0,0,7.8292
--1,3,1,44,0,0,8.05
-1,3,1,,0,0,7.75
--1,3,1,,1,0,14.4542
--1,3,2,,1,0,14.4542
--1,3,1,,1,0,7.75
--1,3,1,,1,0,7.75
--1,3,1,,0,0,7.7375
--1,3,2,22,2,0,8.6625
--1,3,1,26,2,0,8.6625
-1,3,2,4,0,2,22.025
-1,3,1,29,3,1,22.025
-1,3,2,26,1,1,22.025
--1,3,2,1,1,1,12.1833
--1,3,1,18,1,1,7.8542
--1,3,2,36,0,2,12.1833
--1,3,1,,0,0,7.8958
-1,3,1,25,0,0,7.2292
--1,3,1,,0,0,7.225
--1,3,2,37,0,0,9.5875
--1,3,1,,0,0,7.8958
-1,3,1,,0,0,56.4958
--1,3,1,,0,0,56.4958
-1,3,2,22,0,0,7.25
--1,3,1,,0,0,7.75
-1,3,1,26,0,0,56.4958
--1,3,1,29,0,0,9.4833
--1,3,1,29,0,0,7.775
--1,3,1,22,0,0,7.775
-1,3,1,22,0,0,7.225
--1,3,1,,3,1,25.4667
--1,3,2,,3,1,25.4667
--1,3,2,,3,1,25.4667
--1,3,2,,3,1,25.4667
--1,3,2,,0,4,25.4667
--1,3,1,32,0,0,7.925
--1,3,1,34.5,0,0,6.4375
--1,3,2,,1,0,15.5
--1,3,1,,1,0,15.5
--1,3,1,36,0,0,0
--1,3,1,39,0,0,24.15
--1,3,1,24,0,0,9.5
--1,3,2,25,0,0,7.775
--1,3,2,45,0,0,7.75
--1,3,1,36,1,0,15.55
--1,3,2,30,1,0,15.55
-1,3,1,20,1,0,7.925
--1,3,1,,0,0,7.8792
--1,3,1,28,0,0,56.4958
--1,3,1,,0,0,7.55
--1,3,1,30,1,0,16.1
--1,3,2,26,1,0,16.1
--1,3,1,,0,0,7.8792
--1,3,1,20.5,0,0,7.25
-1,3,1,27,0,0,8.6625
--1,3,1,51,0,0,7.0542
-1,3,2,23,0,0,7.8542
-1,3,1,32,0,0,7.5792
--1,3,1,,0,0,7.8958
--1,3,1,,0,0,7.55
-1,3,2,,0,0,7.75
-1,3,1,24,0,0,7.1417
--1,3,1,22,0,0,7.125
--1,3,2,,0,0,7.8792
--1,3,1,,0,0,7.75
--1,3,1,,0,0,8.05
--1,3,1,29,0,0,7.925
-1,3,1,,0,0,7.2292
--1,3,2,30.5,0,0,7.75
-1,3,2,,0,0,7.7375
--1,3,1,,0,0,7.2292
--1,3,1,35,0,0,7.8958
--1,3,1,33,0,0,7.8958
-1,3,2,,0,0,7.225
--1,3,1,,0,0,7.8958
-1,3,2,,0,0,7.75
-1,3,1,,0,0,7.75
-1,3,2,,2,0,23.25
-1,3,2,,2,0,23.25
-1,3,1,,2,0,23.25
-1,3,2,,0,0,7.7875
--1,3,1,,0,0,15.5
-1,3,2,,0,0,7.8792
-1,3,2,15,0,0,8.0292
--1,3,2,35,0,0,7.75
--1,3,1,,0,0,7.75
--1,3,1,24,1,0,16.1
--1,3,2,19,1,0,16.1
--1,3,2,,0,0,7.75
--1,3,2,,0,0,8.05
--1,3,2,,0,0,8.05
--1,3,1,55.5,0,0,8.05
--1,3,1,,0,0,7.75
-1,3,1,21,0,0,7.775
--1,3,1,,0,0,8.05
--1,3,1,24,0,0,7.8958
--1,3,1,21,0,0,7.8958
--1,3,1,28,0,0,7.8958
--1,3,1,,0,0,7.8958
-1,3,2,,0,0,7.8792
--1,3,1,25,0,0,7.65
-1,3,1,6,0,1,12.475
-1,3,2,27,0,1,12.475
--1,3,1,,0,0,8.05
-1,3,2,,1,0,24.15
--1,3,1,,1,0,24.15
--1,3,1,,0,0,8.4583
--1,3,1,34,0,0,8.05
--1,3,1,,0,0,7.75
-1,3,1,,0,0,7.775
-1,3,1,,1,1,15.2458
-1,3,1,,1,1,15.2458
-1,3,2,,0,2,15.2458
-1,3,2,,0,0,7.2292
--1,3,1,,0,0,8.05
-1,3,2,,0,0,7.7333
-1,3,2,24,0,0,7.75
--1,3,1,,0,0,8.05
-1,3,2,,1,0,15.5
-1,3,2,,1,0,15.5
-1,3,2,,0,0,15.5
--1,3,1,18,0,0,7.75
--1,3,1,22,0,0,7.8958
-1,3,2,15,0,0,7.225
-1,3,2,1,0,2,15.7417
-1,3,1,20,1,1,15.7417
-1,3,2,19,1,1,15.7417
--1,3,1,33,0,0,8.05
--1,3,1,,0,0,7.8958
--1,3,1,,0,0,7.2292
--1,3,2,,0,0,7.75
--1,3,1,,0,0,7.8958
-1,3,1,12,1,0,11.2417
-1,3,2,14,1,0,11.2417
--1,3,2,29,0,0,7.925
--1,3,1,28,0,0,8.05
-1,3,2,18,0,0,7.775
-1,3,2,26,0,0,7.8542
--1,3,1,21,0,0,7.8542
--1,3,1,41,0,0,7.125
-1,3,1,39,0,0,7.925
--1,3,1,21,0,0,7.8
--1,3,1,28.5,0,0,7.2292
-1,3,2,22,0,0,7.75
--1,3,1,61,0,0,6.2375
--1,3,1,,1,0,15.5
--1,3,1,,0,0,7.8292
-1,3,2,,1,0,15.5
--1,3,1,,0,0,7.7333
--1,3,1,,0,0,7.75
--1,3,1,,0,0,7.75
--1,3,1,23,0,0,9.225
--1,3,2,,0,0,7.75
-1,3,2,,0,0,7.75
-1,3,2,,0,0,7.8792
-1,3,2,22,0,0,7.775
-1,3,1,,0,0,7.75
-1,3,2,,0,0,7.8292
-1,3,1,9,0,1,3.1708
--1,3,1,28,0,0,22.525
--1,3,1,42,0,1,8.4042
--1,3,1,,0,0,7.3125
--1,3,2,31,0,0,7.8542
--1,3,1,28,0,0,7.8542
-1,3,1,32,0,0,7.775
--1,3,1,20,0,0,9.225
--1,3,2,23,0,0,8.6625
--1,3,2,20,0,0,8.6625
--1,3,1,20,0,0,8.6625
--1,3,1,16,0,0,9.2167
-1,3,2,31,0,0,8.6833
--1,3,2,,0,0,7.6292
--1,3,1,2,3,1,21.075
--1,3,1,6,3,1,21.075
--1,3,2,3,3,1,21.075
--1,3,2,8,3,1,21.075
--1,3,2,29,0,4,21.075
--1,3,1,1,4,1,39.6875
--1,3,1,7,4,1,39.6875
--1,3,1,2,4,1,39.6875
--1,3,1,16,4,1,39.6875
--1,3,1,14,4,1,39.6875
--1,3,2,41,0,5,39.6875
--1,3,1,21,0,0,8.6625
--1,3,1,19,0,0,14.5
--1,3,1,,0,0,8.7125
--1,3,1,32,0,0,7.8958
--1,3,1,0.75,1,1,13.775
--1,3,2,3,1,1,13.775
--1,3,2,26,0,2,13.775
--1,3,1,,0,0,7
--1,3,1,,0,0,7.775
--1,3,1,,0,0,8.05
--1,3,1,21,0,0,7.925
--1,3,1,25,0,0,7.925
--1,3,1,22,0,0,7.25
-1,3,1,25,1,0,7.775
-1,3,1,,1,1,22.3583
-1,3,2,,1,1,22.3583
-1,3,2,,0,2,22.3583
--1,3,2,,0,0,8.1375
--1,3,1,24,0,0,8.05
--1,3,2,28,0,0,7.8958
--1,3,1,19,0,0,7.8958
--1,3,1,,0,0,7.8958
--1,3,1,25,1,0,7.775
--1,3,2,18,0,0,7.775
-1,3,1,32,0,0,8.05
--1,3,1,,0,0,7.8958
--1,3,1,17,0,0,8.6625
--1,3,1,24,0,0,8.6625
--1,3,1,,0,0,7.8958
--1,3,2,,0,0,8.1125
--1,3,1,,0,0,7.2292
--1,3,1,,0,0,7.25
--1,3,1,38,0,0,7.8958
--1,3,1,21,0,0,8.05
--1,3,1,10,4,1,29.125
--1,3,1,4,4,1,29.125
--1,3,1,7,4,1,29.125
--1,3,1,2,4,1,29.125
--1,3,1,8,4,1,29.125
--1,3,2,39,0,5,29.125
--1,3,2,22,0,0,39.6875
--1,3,1,35,0,0,7.125
-1,3,2,,0,0,7.7208
--1,3,1,,0,0,14.5
--1,3,2,,0,0,14.5
--1,3,1,50,1,0,14.5
--1,3,2,47,1,0,14.5
--1,3,1,,0,0,8.05
--1,3,1,,0,0,7.775
--1,3,2,2,1,1,20.2125
--1,3,1,18,1,1,20.2125
--1,3,2,41,0,2,20.2125
-1,3,2,,0,0,8.05
--1,3,1,50,0,0,8.05
--1,3,1,16,0,0,8.05
-1,3,1,,0,0,7.75
--1,3,1,,0,0,24.15
--1,3,1,,0,0,7.2292
--1,3,1,25,0,0,7.225
--1,3,1,,0,0,7.225
--1,3,1,,0,0,7.7292
--1,3,1,,0,0,7.575
--1,3,1,38.5,0,0,7.25
--1,3,1,,8,2,69.55
--1,3,1,14.5,8,2,69.55
--1,3,2,,8,2,69.55
--1,3,2,,8,2,69.55
--1,3,2,,8,2,69.55
--1,3,2,,8,2,69.55
--1,3,1,,8,2,69.55
--1,3,1,,8,2,69.55
--1,3,1,,8,2,69.55
--1,3,1,,1,9,69.55
--1,3,2,,1,9,69.55
--1,3,1,24,0,0,9.325
-1,3,2,21,0,0,7.65
--1,3,1,39,0,0,7.925
--1,3,1,,2,0,21.6792
--1,3,1,,2,0,21.6792
--1,3,1,,2,0,21.6792
-1,3,2,1,1,1,16.7
-1,3,2,24,0,2,16.7
-1,3,2,4,1,1,16.7
-1,3,1,25,0,0,9.5
--1,3,1,20,0,0,8.05
--1,3,1,24.5,0,0,8.05
--1,3,1,,0,0,7.725
--1,3,1,,0,0,7.8958
--1,3,1,,0,0,7.75
-1,3,1,29,0,0,9.5
--1,3,1,,0,0,15.1
-1,3,2,,0,0,7.7792
--1,3,1,,0,0,8.05
--1,3,1,,0,0,8.05
--1,3,1,22,0,0,7.2292
--1,3,1,,0,0,8.05
--1,3,1,40,0,0,7.8958
--1,3,1,21,0,0,7.925
-1,3,2,18,0,0,7.4958
--1,3,1,4,3,2,27.9
--1,3,1,10,3,2,27.9
--1,3,2,9,3,2,27.9
--1,3,2,2,3,2,27.9
--1,3,1,40,1,4,27.9
--1,3,2,45,1,4,27.9
--1,3,1,,0,0,7.8958
--1,3,1,,0,0,8.05
--1,3,1,,0,0,8.6625
--1,3,1,,0,0,7.75
-1,3,2,,0,0,7.7333
--1,3,1,19,0,0,7.65
--1,3,1,30,0,0,8.05
--1,3,1,,0,0,8.05
--1,3,1,32,0,0,8.05
--1,3,1,,0,0,7.8958
--1,3,1,33,0,0,8.6625
-1,3,2,23,0,0,7.55
--1,3,1,21,0,0,8.05
--1,3,1,60.5,0,0,0
--1,3,1,19,0,0,7.8958
--1,3,2,22,0,0,9.8375
-1,3,1,31,0,0,7.925
--1,3,1,27,0,0,8.6625
--1,3,2,2,0,1,10.4625
--1,3,2,29,1,1,10.4625
-1,3,1,16,0,0,8.05
-1,3,1,44,0,0,7.925
--1,3,1,25,0,0,7.05
--1,3,1,74,0,0,7.775
-1,3,1,14,0,0,9.225
--1,3,1,24,0,0,7.7958
-1,3,1,25,0,0,7.7958
--1,3,1,34,0,0,8.05
-1,3,1,0.4167,0,1,8.5167
--1,3,1,,1,0,6.4375
--1,3,1,,0,0,6.4375
--1,3,1,,0,0,7.225
-1,3,2,16,1,1,8.5167
--1,3,1,,0,0,8.05
--1,3,1,,1,0,16.1
-1,3,2,,1,0,16.1
--1,3,1,32,0,0,7.925
--1,3,1,,0,0,7.75
--1,3,1,,0,0,7.8958
--1,3,1,30.5,0,0,8.05
--1,3,1,44,0,0,8.05
--1,3,1,,0,0,7.2292
-1,3,1,25,0,0,0
--1,3,1,,0,0,7.2292
-1,3,1,7,1,1,15.2458
-1,3,2,9,1,1,15.2458
-1,3,2,29,0,2,15.2458
--1,3,1,36,0,0,7.8958
-1,3,2,18,0,0,9.8417
-1,3,2,63,0,0,9.5875
--1,3,1,,1,1,14.5
--1,3,1,11.5,1,1,14.5
--1,3,1,40.5,0,2,14.5
--1,3,2,10,0,2,24.15
--1,3,1,36,1,1,24.15
--1,3,2,30,1,1,24.15
--1,3,1,,0,0,9.5
--1,3,1,33,0,0,9.5
--1,3,1,28,0,0,9.5
--1,3,1,28,0,0,9.5
--1,3,1,47,0,0,9
--1,3,2,18,2,0,18
--1,3,1,31,3,0,18
--1,3,1,16,2,0,18
--1,3,2,31,1,0,18
-1,3,1,22,0,0,7.225
--1,3,1,20,0,0,7.8542
--1,3,2,14,0,0,7.8542
--1,3,1,22,0,0,7.8958
--1,3,1,22,0,0,9
--1,3,1,,0,0,8.05
--1,3,1,,0,0,7.55
--1,3,1,,0,0,8.05
--1,3,1,32.5,0,0,9.5
-1,3,2,38,0,0,7.2292
--1,3,1,51,0,0,7.75
--1,3,1,18,1,0,6.4958
--1,3,1,21,1,0,6.4958
-1,3,2,47,1,0,7
--1,3,1,,0,0,8.7125
--1,3,1,,0,0,7.55
--1,3,1,,0,0,8.05
--1,3,1,28.5,0,0,16.1
--1,3,1,21,0,0,7.25
--1,3,1,27,0,0,8.6625
--1,3,1,,0,0,7.25
--1,3,1,36,0,0,9.5
--1,3,1,27,1,0,14.4542
-1,3,2,15,1,0,14.4542
--1,3,1,45.5,0,0,7.225
--1,3,1,,0,0,7.225
--1,3,1,,0,0,14.4583
--1,3,2,14.5,1,0,14.4542
--1,3,2,,1,0,14.4542
--1,3,1,26.5,0,0,7.225
--1,3,1,27,0,0,7.225
--1,3,1,29,0,0,7.875
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/structures/Dataset.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/structures/Dataset.java b/modules/ml/src/main/java/org/apache/ignite/ml/structures/Dataset.java
index 89ff1b4..cbed297 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/structures/Dataset.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/structures/Dataset.java
@@ -86,6 +86,16 @@ public class Dataset<Row extends DatasetRow> implements Serializable, Externaliz
}
/**
+ * Creates new Dataset by given data.
+ *
+ * @param data Should be initialized with one vector at least.
+ */
+ public Dataset(Row[] data) {
+ this.data = data;
+ this.rowSize = data.length;
+ }
+
+ /**
* Creates new Dataset and initialized with empty data structure.
*
* @param rowSize Amount of instances. Should be > 0.
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java b/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java
index 0028a16..3239116 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java
@@ -27,7 +27,7 @@ import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector;
/**
* Class for set of labeled vectors.
*/
-public class LabeledDataset<L, Row extends LabeledVector> extends Dataset<Row> {
+public class LabeledDataset<L, Row extends LabeledVector> extends Dataset<Row> implements AutoCloseable {
/**
* Default constructor (required by Externalizable).
*/
@@ -70,6 +70,15 @@ public class LabeledDataset<L, Row extends LabeledVector> extends Dataset<Row> {
initializeDataWithLabeledVectors();
}
+ /**
+ * Creates new Labeled Dataset by given data.
+ *
+ * @param data Should be initialized with one vector at least.
+ */
+ public LabeledDataset(Row[] data) {
+ super(data);
+ }
+
/** */
private void initializeDataWithLabeledVectors() {
data = (Row[])new LabeledVector[rowSize];
@@ -207,4 +216,9 @@ public class LabeledDataset<L, Row extends LabeledVector> extends Dataset<Row> {
return res;
}
+
+ /** Closes LabeledDataset. */
+ @Override public void close() throws Exception {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java
index 6d93402..4fd2e0e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java
@@ -146,7 +146,9 @@ public class SVMLinearBinaryClassificationModel implements Model<Vector, Double>
return true;
if (o == null || getClass() != o.getClass())
return false;
+
SVMLinearBinaryClassificationModel mdl = (SVMLinearBinaryClassificationModel)o;
+
return Double.compare(mdl.intercept, intercept) == 0
&& Double.compare(mdl.threshold, threshold) == 0
&& Boolean.compare(mdl.isKeepingRawLabels, isKeepingRawLabels) == 0
@@ -176,7 +178,7 @@ public class SVMLinearBinaryClassificationModel implements Model<Vector, Double>
return builder.toString();
}
- return "LinearRegressionModel{" +
+ return "SVMModel{" +
"weights=" + weights +
", intercept=" + intercept +
'}';
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
index ee3b6e8..e745ca9 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
@@ -18,10 +18,13 @@
package org.apache.ignite.ml.svm;
import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.ml.Trainer;
+import org.apache.ignite.ml.DatasetTrainer;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.PartitionDataBuilder;
import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector;
import org.apache.ignite.ml.structures.LabeledDataset;
import org.apache.ignite.ml.structures.LabeledVector;
import org.jetbrains.annotations.NotNull;
@@ -32,33 +35,50 @@ import org.jetbrains.annotations.NotNull;
* and +1 labels for two classes and makes binary classification. </p> The paper about this algorithm could be found
* here https://arxiv.org/abs/1409.1458.
*/
-public class SVMLinearBinaryClassificationTrainer implements Trainer<SVMLinearBinaryClassificationModel, LabeledDataset> {
+public class SVMLinearBinaryClassificationTrainer<K, V>
+ implements DatasetTrainer<K, V, SVMLinearBinaryClassificationModel> {
/** Amount of outer SDCA algorithm iterations. */
- private int amountOfIterations = 20;
+ private int amountOfIterations = 200;
/** Amount of local SDCA algorithm iterations. */
- private int amountOfLocIterations = 50;
+ private int amountOfLocIterations = 100;
/** Regularization parameter. */
- private double lambda = 0.2;
+ private double lambda = 0.4;
- /** This flag enables distributed mode for this algorithm. */
- private boolean isDistributed;
+ /** Dataset. */
+ private Dataset<SVMPartitionContext, LabeledDataset<Double, LabeledVector>> dataset;
/**
- * Returns model based on data
+ * Trains model based on the specified data.
*
- * @param data data to build model
- * @return model
+ * @param datasetBuilder Dataset builder.
+ * @param featureExtractor Feature extractor.
+ * @param lbExtractor Label extractor.
+ * @param cols Number of columns.
+ * @return Model.
*/
- @Override public SVMLinearBinaryClassificationModel train(LabeledDataset data) {
- isDistributed = data.isDistributed();
+ @Override public SVMLinearBinaryClassificationModel fit(DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor, int cols) {
- final int weightVectorSizeWithIntercept = data.colSize() + 1;
+ assert datasetBuilder != null;
+
+ PartitionDataBuilder<K, V, SVMPartitionContext, LabeledDataset<Double, LabeledVector>> partDataBuilder = new SVMPartitionDataBuilderOnHeap<>(
+ featureExtractor,
+ lbExtractor,
+ cols
+ );
+
+ this.dataset = datasetBuilder.build(
+ (upstream, upstreamSize) -> new SVMPartitionContext(),
+ partDataBuilder
+ );
+
+
+ final int weightVectorSizeWithIntercept = cols + 1;
Vector weights = initializeWeightsWithZeros(weightVectorSizeWithIntercept);
for (int i = 0; i < this.getAmountOfIterations(); i++) {
- Vector deltaWeights = calculateUpdates(data, weights);
+ Vector deltaWeights = calculateUpdates(weights);
weights = weights.plus(deltaWeights); // creates new vector
}
@@ -67,34 +87,32 @@ public class SVMLinearBinaryClassificationTrainer implements Trainer<SVMLinearBi
/** */
@NotNull private Vector initializeWeightsWithZeros(int vectorSize) {
- if (isDistributed)
- return new SparseDistributedVector(vectorSize);
- else
return new DenseLocalOnHeapVector(vectorSize);
}
/** */
- private Vector calculateUpdates(LabeledDataset data, Vector weights) {
- Vector copiedWeights = weights.copy();
- Vector deltaWeights = initializeWeightsWithZeros(weights.size());
-
- final int amountOfObservation = data.rowSize();
+ private Vector calculateUpdates(Vector weights) {
+ return dataset.compute(data -> {
+ Vector copiedWeights = weights.copy();
+ Vector deltaWeights = initializeWeightsWithZeros(weights.size());
+ final int amountOfObservation = data.rowSize();
- Vector tmpAlphas = initializeWeightsWithZeros(amountOfObservation);
- Vector deltaAlphas = initializeWeightsWithZeros(amountOfObservation);
+ Vector tmpAlphas = initializeWeightsWithZeros(amountOfObservation);
+ Vector deltaAlphas = initializeWeightsWithZeros(amountOfObservation);
- for (int i = 0; i < this.getAmountOfLocIterations(); i++) {
- int randomIdx = ThreadLocalRandom.current().nextInt(amountOfObservation);
+ for (int i = 0; i < this.getAmountOfLocIterations(); i++) {
+ int randomIdx = ThreadLocalRandom.current().nextInt(amountOfObservation);
- Deltas deltas = getDeltas(data, copiedWeights, amountOfObservation, tmpAlphas, randomIdx);
+ Deltas deltas = getDeltas(data, copiedWeights, amountOfObservation, tmpAlphas, randomIdx);
- copiedWeights = copiedWeights.plus(deltas.deltaWeights); // creates new vector
- deltaWeights = deltaWeights.plus(deltas.deltaWeights); // creates new vector
+ copiedWeights = copiedWeights.plus(deltas.deltaWeights); // creates new vector
+ deltaWeights = deltaWeights.plus(deltas.deltaWeights); // creates new vector
- tmpAlphas.set(randomIdx, tmpAlphas.get(randomIdx) + deltas.deltaAlpha);
- deltaAlphas.set(randomIdx, deltaAlphas.get(randomIdx) + deltas.deltaAlpha);
- }
- return deltaWeights;
+ tmpAlphas.set(randomIdx, tmpAlphas.get(randomIdx) + deltas.deltaAlpha);
+ deltaAlphas.set(randomIdx, deltaAlphas.get(randomIdx) + deltas.deltaAlpha);
+ }
+ return deltaWeights;
+ }, (a, b) -> a == null ? b : a.plus(b));
}
/** */
@@ -225,6 +243,7 @@ public class SVMLinearBinaryClassificationTrainer implements Trainer<SVMLinearBi
this.amountOfLocIterations = amountOfLocIterations;
return this;
}
+
}
/** This is a helper class to handle pair results which are returned from the calculation method. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
deleted file mode 100644
index fd91595..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import java.util.TreeMap;
-import org.apache.ignite.ml.Exportable;
-import org.apache.ignite.ml.Exporter;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.math.Vector;
-
-/** Base class for multi-classification model for set of SVM classifiers. */
-public class SVMLinearMultiClassClassificationModel implements Model<Vector, Double>, Exportable<SVMLinearMultiClassClassificationModel>, Serializable {
- /** List of models associated with each class. */
- private Map<Double, SVMLinearBinaryClassificationModel> models;
-
- /** */
- public SVMLinearMultiClassClassificationModel() {
- this.models = new HashMap<>();
- }
-
- /** {@inheritDoc} */
- @Override public Double apply(Vector input) {
- TreeMap<Double, Double> maxMargins = new TreeMap<>();
-
- models.forEach((k, v) -> maxMargins.put(input.dot(v.weights()) + v.intercept(), k));
-
- return maxMargins.lastEntry().getValue();
- }
-
- /** {@inheritDoc} */
- @Override public <P> void saveModel(Exporter<SVMLinearMultiClassClassificationModel, P> exporter, P path) {
- exporter.save(this, path);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
- SVMLinearMultiClassClassificationModel mdl = (SVMLinearMultiClassClassificationModel)o;
- return Objects.equals(models, mdl.models);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return Objects.hash(models);
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- StringBuilder wholeStr = new StringBuilder();
-
- models.forEach((clsLb, mdl) -> {
- wholeStr.append("The class with label " + clsLb + " has classifier: " + mdl.toString() + System.lineSeparator());
- });
-
- return wholeStr.toString();
- }
-
- /**
- * Adds a specific SVM binary classifier to the bunch of same classifiers.
- *
- * @param clsLb The class label for the added model.
- * @param mdl The model.
- */
- public void add(double clsLb, SVMLinearBinaryClassificationModel mdl) {
- models.put(clsLb, mdl);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
deleted file mode 100644
index 669e2e3..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.structures.LabeledDataset;
-
-/**
- * Base class for a soft-margin SVM linear multiclass-classification trainer based on the communication-efficient
- * distributed dual coordinate ascent algorithm (CoCoA) with hinge-loss function.
- *
- * All common parameters are shared with bunch of binary classification trainers.
- */
-public class SVMLinearMultiClassClassificationTrainer implements Trainer<SVMLinearMultiClassClassificationModel, LabeledDataset> {
- /** Amount of outer SDCA algorithm iterations. */
- private int amountOfIterations = 20;
-
- /** Amount of local SDCA algorithm iterations. */
- private int amountOfLocIterations = 50;
-
- /** Regularization parameter. */
- private double lambda = 0.2;
-
- /**
- * Returns model based on data.
- *
- * @param data data to build model.
- * @return model.
- */
- @Override public SVMLinearMultiClassClassificationModel train(LabeledDataset data) {
- List<Double> classes = getClassLabels(data);
-
- SVMLinearMultiClassClassificationModel multiClsMdl = new SVMLinearMultiClassClassificationModel();
-
- classes.forEach(clsLb -> {
- LabeledDataset binarizedDataset = binarizeLabels(data, clsLb);
-
- SVMLinearBinaryClassificationTrainer trainer = new SVMLinearBinaryClassificationTrainer()
- .withAmountOfIterations(this.amountOfIterations())
- .withAmountOfLocIterations(this.amountOfLocIterations())
- .withLambda(this.lambda());
-
- multiClsMdl.add(clsLb, trainer.train(binarizedDataset));
- });
-
- return multiClsMdl;
- }
-
- /**
- * Copies the given data and changes class labels in +1 for chosen class and in -1 for the rest classes.
- *
- * @param data Data to transform.
- * @param clsLb Chosen class in schema One-vs-Rest.
- * @return Copy of dataset with new labels.
- */
- private LabeledDataset binarizeLabels(LabeledDataset data, double clsLb) {
- final LabeledDataset ds = data.copy();
-
- for (int i = 0; i < ds.rowSize(); i++)
- ds.setLabel(i, ds.label(i) == clsLb ? 1.0 : -1.0);
-
- return ds;
- }
-
- /** Iterates among dataset and collects class labels. */
- private List<Double> getClassLabels(LabeledDataset data) {
- final Set<Double> clsLabels = new HashSet<>();
-
- for (int i = 0; i < data.rowSize(); i++)
- clsLabels.add(data.label(i));
-
- List<Double> res = new ArrayList<>();
- res.addAll(clsLabels);
-
- return res;
- }
-
- /**
- * Set up the regularization parameter.
- *
- * @param lambda The regularization parameter. Should be more than 0.0.
- * @return Trainer with new lambda parameter value.
- */
- public SVMLinearMultiClassClassificationTrainer withLambda(double lambda) {
- assert lambda > 0.0;
- this.lambda = lambda;
- return this;
- }
-
- /**
- * Gets the regularization lambda.
- *
- * @return The parameter value.
- */
- public double lambda() {
- return lambda;
- }
-
- /**
- * Gets the amount of outer iterations of SCDA algorithm.
- *
- * @return The parameter value.
- */
- public int amountOfIterations() {
- return amountOfIterations;
- }
-
- /**
- * Set up the amount of outer iterations of SCDA algorithm.
- *
- * @param amountOfIterations The parameter value.
- * @return Trainer with new amountOfIterations parameter value.
- */
- public SVMLinearMultiClassClassificationTrainer withAmountOfIterations(int amountOfIterations) {
- this.amountOfIterations = amountOfIterations;
- return this;
- }
-
- /**
- * Gets the amount of local iterations of SCDA algorithm.
- *
- * @return The parameter value.
- */
- public int amountOfLocIterations() {
- return amountOfLocIterations;
- }
-
- /**
- * Set up the amount of local iterations of SCDA algorithm.
- *
- * @param amountOfLocIterations The parameter value.
- * @return Trainer with new amountOfLocIterations parameter value.
- */
- public SVMLinearMultiClassClassificationTrainer withAmountOfLocIterations(int amountOfLocIterations) {
- this.amountOfLocIterations = amountOfLocIterations;
- return this;
- }
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionContext.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionContext.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionContext.java
new file mode 100644
index 0000000..0aee0fb
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionContext.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.svm;
+
+import java.io.Serializable;
+
+/**
+ * Partition context of the SVM classification algorithm.
+ */
+public class SVMPartitionContext implements Serializable {
+ /** */
+ private static final long serialVersionUID = -7212307112344430126L;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionDataBuilderOnHeap.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionDataBuilderOnHeap.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionDataBuilderOnHeap.java
new file mode 100644
index 0000000..ad85758
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionDataBuilderOnHeap.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.svm;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import org.apache.ignite.ml.dataset.PartitionDataBuilder;
+import org.apache.ignite.ml.dataset.UpstreamEntry;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.structures.LabeledDataset;
+import org.apache.ignite.ml.structures.LabeledVector;
+
+/**
+ * SVM partition data builder that builds {@link LabeledDataset}.
+ *
+ * @param <K> Type of a key in <tt>upstream</tt> data.
+ * @param <V> Type of a value in <tt>upstream</tt> data.
+ * @param <C> Type of a partition <tt>context</tt>.
+ */
+public class SVMPartitionDataBuilderOnHeap<K, V, C extends Serializable>
+ implements PartitionDataBuilder<K, V, C, LabeledDataset<Double, LabeledVector>> {
+ /** */
+ private static final long serialVersionUID = -7820760153954269227L;
+
+ /** Extractor of X matrix row. */
+ private final IgniteBiFunction<K, V, double[]> xExtractor;
+
+ /** Extractor of Y vector value. */
+ private final IgniteBiFunction<K, V, Double> yExtractor;
+
+ /** Number of columns. */
+ private final int cols;
+
+ /**
+ * Constructs a new instance of SVM partition data builder.
+ *
+ * @param xExtractor Extractor of X matrix row.
+ * @param yExtractor Extractor of Y vector value.
+ * @param cols Number of columns.
+ */
+ public SVMPartitionDataBuilderOnHeap(IgniteBiFunction<K, V, double[]> xExtractor,
+ IgniteBiFunction<K, V, Double> yExtractor, int cols) {
+ this.xExtractor = xExtractor;
+ this.yExtractor = yExtractor;
+ this.cols = cols;
+ }
+
+ /** {@inheritDoc} */
+ @Override public LabeledDataset<Double, LabeledVector> build(Iterator<UpstreamEntry<K, V>> upstreamData, long upstreamDataSize,
+ C ctx) {
+
+ double[][] x = new double[Math.toIntExact(upstreamDataSize)][cols];
+ double[] y = new double[Math.toIntExact(upstreamDataSize)];
+
+ int ptr = 0;
+ while (upstreamData.hasNext()) {
+ UpstreamEntry<K, V> entry = upstreamData.next();
+ double[] row = xExtractor.apply(entry.getKey(), entry.getValue());
+
+ assert row.length == cols : "X extractor must return exactly " + cols + " columns";
+
+ x[ptr] = row;
+
+ y[ptr] = yExtractor.apply(entry.getKey(), entry.getValue());
+
+ ptr++;
+ }
+
+ return new LabeledDataset<>(x, y);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java
index 57d93d6..421a19f 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java
@@ -32,7 +32,6 @@ import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
import org.apache.ignite.ml.structures.LabeledDataset;
import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
-import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationModel;
import org.junit.Assert;
import org.junit.Test;
@@ -98,33 +97,6 @@ public class LocalModelsTest {
/** */
- @Test
- public void importExportSVMMulticlassClassificationModelTest() throws IOException {
- executeModelTest(mdlFilePath -> {
-
-
- SVMLinearBinaryClassificationModel binaryMdl1 = new SVMLinearBinaryClassificationModel(new DenseLocalOnHeapVector(new double[]{1, 2}), 3);
- SVMLinearBinaryClassificationModel binaryMdl2 = new SVMLinearBinaryClassificationModel(new DenseLocalOnHeapVector(new double[]{2, 3}), 4);
- SVMLinearBinaryClassificationModel binaryMdl3 = new SVMLinearBinaryClassificationModel(new DenseLocalOnHeapVector(new double[]{3, 4}), 5);
-
- SVMLinearMultiClassClassificationModel mdl = new SVMLinearMultiClassClassificationModel();
- mdl.add(1, binaryMdl1);
- mdl.add(2, binaryMdl2);
- mdl.add(3, binaryMdl3);
-
- Exporter<SVMLinearMultiClassClassificationModel, String> exporter = new FileExporter<>();
- mdl.saveModel(exporter, mdlFilePath);
-
- SVMLinearMultiClassClassificationModel load = exporter.load(mdlFilePath);
-
- Assert.assertNotNull(load);
- Assert.assertEquals("", mdl, load);
-
- return null;
- });
- }
-
- /** */
private void executeModelTest(Function<String, Void> code) throws IOException {
Path mdlPath = Files.createTempFile(null, null);
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/BaseSVMTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/BaseSVMTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/BaseSVMTest.java
deleted file mode 100644
index 424118d..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/BaseSVMTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Base class for SVM tests.
- */
-public class BaseSVMTest extends GridCommonAbstractTest {
- /** Count of nodes. */
- private static final int NODE_COUNT = 4;
-
- /** Grid instance. */
- protected Ignite ignite;
-
- /**
- * Default constructor.
- */
- public BaseSVMTest() {
- super(false);
- }
-
- /**
- * {@inheritDoc}
- */
- @Override protected void beforeTest() throws Exception {
- ignite = grid(NODE_COUNT);
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- for (int i = 1; i <= NODE_COUNT; i++)
- startGrid(i);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- stopAllGrids();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
new file mode 100644
index 0000000..353915c
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.svm;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.ml.TestUtils;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.junit.Test;
+
+/**
+ * Tests for {@link SVMLinearBinaryClassificationTrainer}.
+ */
+public class SVMBinaryTrainerTest {
+ /** Fixed size of Dataset. */
+ private static final int AMOUNT_OF_OBSERVATIONS = 1000;
+
+ /** Fixed size of columns in Dataset. */
+ private static final int AMOUNT_OF_FEATURES = 2;
+
+ /**
+ * Test trainer on classification model y = x.
+ */
+ @Test
+ public void testTrainWithTheLinearlySeparableCase() {
+ Map<Integer, double[]> data = new HashMap<>();
+
+
+ ThreadLocalRandom rndX = ThreadLocalRandom.current();
+ ThreadLocalRandom rndY = ThreadLocalRandom.current();
+
+ for (int i = 0; i < AMOUNT_OF_OBSERVATIONS; i++) {
+ double x = rndX.nextDouble(-1000, 1000);
+ double y = rndY.nextDouble(-1000, 1000);
+ double[] vec = new double[AMOUNT_OF_FEATURES + 1];
+ vec[0] = y - x > 0 ? 1 : -1; // assign label.
+ vec[1] = x;
+ vec[2] = y;
+ data.put(i, vec);
+ }
+
+
+ SVMLinearBinaryClassificationTrainer<Integer, double[]> trainer = new SVMLinearBinaryClassificationTrainer<>();
+
+ SVMLinearBinaryClassificationModel mdl = trainer.fit(
+ new LocalDatasetBuilder<>(data, 10),
+ (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+ (k, v) -> v[0],
+ AMOUNT_OF_FEATURES);
+
+ double precision = 1e-2;
+
+ TestUtils.assertEquals(-1, mdl.apply(new DenseLocalOnHeapVector(new double[]{100, 10})), precision);
+ TestUtils.assertEquals(1, mdl.apply(new DenseLocalOnHeapVector(new double[]{10, 100})), precision);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMModelTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMModelTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMModelTest.java
index 35b6644..2533466 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMModelTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMModelTest.java
@@ -59,21 +59,6 @@ public class SVMModelTest {
/** */
@Test
- public void testPredictWithMultiClasses() {
- Vector weights1 = new DenseLocalOnHeapVector(new double[]{10.0, 0.0});
- Vector weights2 = new DenseLocalOnHeapVector(new double[]{0.0, 10.0});
- Vector weights3 = new DenseLocalOnHeapVector(new double[]{-1.0, -1.0});
- SVMLinearMultiClassClassificationModel mdl = new SVMLinearMultiClassClassificationModel();
- mdl.add(1, new SVMLinearBinaryClassificationModel(weights1, 0.0).withRawLabels(true));
- mdl.add(2, new SVMLinearBinaryClassificationModel(weights2, 0.0).withRawLabels(true));
- mdl.add(2, new SVMLinearBinaryClassificationModel(weights3, 0.0).withRawLabels(true));
-
- Vector observation = new DenseLocalOnHeapVector(new double[]{1.0, 1.0});
- TestUtils.assertEquals( 1.0, mdl.apply(observation), PRECISION);
- }
-
- /** */
- @Test
public void testPredictWithErasedLabels() {
Vector weights = new DenseLocalOnHeapVector(new double[]{1.0, 1.0});
SVMLinearBinaryClassificationModel mdl = new SVMLinearBinaryClassificationModel(weights, 1.0);
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMTestSuite.java
index 853a43f..dd87fec 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMTestSuite.java
@@ -17,23 +17,16 @@
package org.apache.ignite.ml.svm;
-import org.apache.ignite.ml.svm.binary.DistributedLinearSVMBinaryClassificationTrainerTest;
-import org.apache.ignite.ml.svm.binary.LocalLinearSVMBinaryClassificationTrainerTest;
-import org.apache.ignite.ml.svm.multi.DistributedLinearSVMMultiClassClassificationTrainerTest;
-import org.apache.ignite.ml.svm.multi.LocalLinearSVMMultiClassClassificationTrainerTest;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
/**
- * Test suite for all tests located in org.apache.ignite.ml.regressions.* package.
+ * Test suite for all tests located in org.apache.ignite.ml.svm.* package.
*/
@RunWith(Suite.class)
@Suite.SuiteClasses({
- LocalLinearSVMBinaryClassificationTrainerTest.class,
- DistributedLinearSVMBinaryClassificationTrainerTest.class,
- LocalLinearSVMMultiClassClassificationTrainerTest.class,
- DistributedLinearSVMMultiClassClassificationTrainerTest.class,
- SVMModelTest.class
+ SVMModelTest.class,
+ SVMBinaryTrainerTest.class
})
public class SVMTestSuite {
// No-op.
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/DistributedLinearSVMBinaryClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/DistributedLinearSVMBinaryClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/DistributedLinearSVMBinaryClassificationTrainerTest.java
deleted file mode 100644
index 1be1d1c..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/DistributedLinearSVMBinaryClassificationTrainerTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm.binary;
-
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
-import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationTrainer;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link DenseLocalOnHeapMatrix}.
- */
-public class DistributedLinearSVMBinaryClassificationTrainerTest extends GenericLinearSVMBinaryClassificationTrainerTest {
- /** */
- public DistributedLinearSVMBinaryClassificationTrainerTest() {
- super(
- new SVMLinearBinaryClassificationTrainer(),
- true,
- 1e-2);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/GenericLinearSVMBinaryClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/GenericLinearSVMBinaryClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/GenericLinearSVMBinaryClassificationTrainerTest.java
deleted file mode 100644
index f390557..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/GenericLinearSVMBinaryClassificationTrainerTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm.binary;
-
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledDataset;
-import org.apache.ignite.ml.structures.LabeledVector;
-import org.apache.ignite.ml.svm.BaseSVMTest;
-import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
-import org.junit.Test;
-
-/**
- * Base class for all linear regression trainers.
- */
-public class GenericLinearSVMBinaryClassificationTrainerTest extends BaseSVMTest {
- /** Fixed size of Dataset. */
- private static final int AMOUNT_OF_OBSERVATIONS = 100;
-
- /** Fixed size of columns in Dataset. */
- private static final int AMOUNT_OF_FEATURES = 2;
-
- /** */
- private final Trainer<SVMLinearBinaryClassificationModel, LabeledDataset> trainer;
-
- /** */
- private boolean isDistributed;
-
- /** */
- private final double precision;
-
- /** */
- GenericLinearSVMBinaryClassificationTrainerTest(
- Trainer<SVMLinearBinaryClassificationModel, LabeledDataset> trainer,
- boolean isDistributed,
- double precision) {
- super();
- this.trainer = trainer;
- this.precision = precision;
- this.isDistributed = isDistributed;
- }
-
- /**
- * Test trainer on classification model y = x.
- */
- @Test
- public void testTrainWithTheLinearlySeparableCase() {
- if (isDistributed)
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
- LabeledDataset dataset = new LabeledDataset<Double, LabeledVector>(AMOUNT_OF_OBSERVATIONS, AMOUNT_OF_FEATURES, isDistributed);
-
- ThreadLocalRandom rndX = ThreadLocalRandom.current();
- ThreadLocalRandom rndY = ThreadLocalRandom.current();
- for (int i = 0; i < AMOUNT_OF_OBSERVATIONS; i++) {
- double x = rndX.nextDouble(-1000, 1000);
- double y = rndY.nextDouble(-1000, 1000);
- dataset.features(i).set(0, x);
- dataset.features(i).set(1, y);
- double lb = y - x > 0 ? 1 : -1;
- dataset.setLabel(i, lb);
- }
-
- SVMLinearBinaryClassificationModel mdl = trainer.train(dataset);
-
- TestUtils.assertEquals(-1, mdl.apply(new DenseLocalOnHeapVector(new double[] {100, 10})), precision);
- TestUtils.assertEquals(1, mdl.apply(new DenseLocalOnHeapVector(new double[] {10, 100})), precision);
- }
-
- /**
- * Test trainer on classification model y = x. Amount of generated points is increased 10 times.
- */
- @Test
- public void testTrainWithTheLinearlySeparableCase10() {
- if (isDistributed)
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
- LabeledDataset dataset = new LabeledDataset<Double, LabeledVector>(AMOUNT_OF_OBSERVATIONS * 10, AMOUNT_OF_FEATURES, isDistributed);
-
- ThreadLocalRandom rndX = ThreadLocalRandom.current();
- ThreadLocalRandom rndY = ThreadLocalRandom.current();
- for (int i = 0; i < AMOUNT_OF_OBSERVATIONS * 10; i++) {
- double x = rndX.nextDouble(-1000, 1000);
- double y = rndY.nextDouble(-1000, 1000);
- dataset.features(i).set(0, x);
- dataset.features(i).set(1, y);
- double lb = y - x > 0 ? 1 : -1;
- dataset.setLabel(i, lb);
- }
-
- SVMLinearBinaryClassificationModel mdl = trainer.train(dataset);
-
- TestUtils.assertEquals(-1, mdl.apply(new DenseLocalOnHeapVector(new double[] {100, 10})), precision);
- TestUtils.assertEquals(1, mdl.apply(new DenseLocalOnHeapVector(new double[] {10, 100})), precision);
- }
-
- /**
- * Test trainer on classification model y = x. Amount of generated points is increased 100 times.
- */
- @Test
- public void testTrainWithTheLinearlySeparableCase100() {
- if (isDistributed)
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
- LabeledDataset dataset = new LabeledDataset<Double, LabeledVector>(AMOUNT_OF_OBSERVATIONS * 100, AMOUNT_OF_FEATURES, isDistributed);
-
- ThreadLocalRandom rndX = ThreadLocalRandom.current();
- ThreadLocalRandom rndY = ThreadLocalRandom.current();
- for (int i = 0; i < AMOUNT_OF_OBSERVATIONS * 100; i++) {
- double x = rndX.nextDouble(-1000, 1000);
- double y = rndY.nextDouble(-1000, 1000);
- dataset.features(i).set(0, x);
- dataset.features(i).set(1, y);
- double lb = y - x > 0 ? 1 : -1;
- dataset.setLabel(i, lb);
- }
-
- SVMLinearBinaryClassificationModel mdl = trainer.train(dataset);
-
- TestUtils.assertEquals(-1, mdl.apply(new DenseLocalOnHeapVector(new double[] {100, 10})), precision);
- TestUtils.assertEquals(1, mdl.apply(new DenseLocalOnHeapVector(new double[] {10, 100})), precision);
- }
-}
[32/50] [abbrv] ignite git commit: IGNITE-7727: Test unmuted. Fixed
by #70ca86a - Fixes #3637.
Posted by sb...@apache.org.
IGNITE-7727: Test unmuted. Fixed by #70ca86a - Fixes #3637.
Signed-off-by: Nikolay Izhikov <ni...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/90e0bf0c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/90e0bf0c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/90e0bf0c
Branch: refs/heads/ignite-zk
Commit: 90e0bf0c1de9b7e372eea52734318f571c5939fa
Parents: 70ca86a
Author: Nikolay Izhikov <ni...@apache.org>
Authored: Wed Mar 14 23:03:26 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Wed Mar 14 23:03:26 2018 +0300
----------------------------------------------------------------------
.../src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/90e0bf0c/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala
index 2ab3681..cd63e8e 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteRDDSpec.scala
@@ -91,8 +91,6 @@ class IgniteRDDSpec extends FunSpec with Matchers with BeforeAndAfterAll with Be
}
it("should successfully store data to ignite using saveValues") {
- fail("https://issues.apache.org/jira/browse/IGNITE-7727")
-
val sc = new SparkContext("local[*]", "test")
try {
@@ -116,8 +114,6 @@ class IgniteRDDSpec extends FunSpec with Matchers with BeforeAndAfterAll with Be
}
it("should successfully store data to ignite using saveValues with inline transformation") {
- fail("https://issues.apache.org/jira/browse/IGNITE-7727")
-
val sc = new SparkContext("local[*]", "test")
try {
[38/50] [abbrv] ignite git commit: IGNITE-6390 Web-console: Clusters
sorted alphabetically.
Posted by sb...@apache.org.
IGNITE-6390 Web-console: Clusters sorted alphabetically.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6d851779
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6d851779
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6d851779
Branch: refs/heads/ignite-zk
Commit: 6d8517796aa1a121d0a6ba5fac7154c05a310eee
Parents: 682197b
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Fri Mar 16 17:05:33 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Mar 16 17:05:33 2018 +0700
----------------------------------------------------------------------
.../frontend/app/components/cluster-selector/controller.js | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6d851779/modules/web-console/frontend/app/components/cluster-selector/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/cluster-selector/controller.js b/modules/web-console/frontend/app/components/cluster-selector/controller.js
index 6a86357..d47432b 100644
--- a/modules/web-console/frontend/app/components/cluster-selector/controller.js
+++ b/modules/web-console/frontend/app/components/cluster-selector/controller.js
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import _ from 'lodash';
+
export default class {
static $inject = ['$scope', 'AgentManager', 'IgniteConfirm'];
@@ -29,7 +31,7 @@ export default class {
this.clusters$ = this.agentMgr.connectionSbj
.do(({ cluster, clusters }) => {
this.cluster = cluster;
- this.clusters = clusters;
+ this.clusters = _.orderBy(clusters, ['name'], ['asc']);
})
.subscribe(() => {});
}
[33/50] [abbrv] ignite git commit: .NET: Fix code inspection errors
Posted by sb...@apache.org.
.NET: Fix code inspection errors
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a69e1acb
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a69e1acb
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a69e1acb
Branch: refs/heads/ignite-zk
Commit: a69e1acba8a69f1818fa9182a1167302f81b93ec
Parents: 90e0bf0
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Thu Mar 15 00:05:32 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Thu Mar 15 00:05:32 2018 +0300
----------------------------------------------------------------------
.../Apache.Ignite.Core/Compute/IComputeTask.cs | 2 ++
.../Query/Continuous/ContinuousQueryFilter.cs | 2 ++
.../Impl/Common/IgniteArgumentCheck.cs | 1 +
.../Impl/Handle/HandleRegistry.cs | 1 +
.../Impl/Unmanaged/Jni/Jvm.cs | 1 +
.../Apache.Ignite.Core/Ssl/SslContextFactory.cs | 8 +++++--
.../Impl/CacheFieldsQueryProvider.cs | 23 +++++++++++---------
7 files changed, 26 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTask.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTask.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTask.cs
index 5ae1420..353e087 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTask.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Compute/IComputeTask.cs
@@ -102,6 +102,8 @@ namespace Apache.Ignite.Core.Compute
/// <param name="rcvd">All previously received results. Note that if task class has
/// <see cref="ComputeTaskNoResultCacheAttribute"/> attribute, then this list will be empty.</param>
/// <returns>Result policy that dictates how to process further upcoming job results.</returns>
+ // ReSharper disable once UnusedParameter.Global
+ // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Global
ComputeJobResultPolicy OnResult(IComputeJobResult<TJobRes> res, IList<IComputeJobResult<TJobRes>> rcvd);
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryFilter.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryFilter.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryFilter.cs
index 7cdb91b..2499463 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryFilter.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/Continuous/ContinuousQueryFilter.cs
@@ -83,6 +83,7 @@ namespace Apache.Ignite.Core.Impl.Cache.Query.Continuous
/** <inheritDoc /> */
public bool Evaluate(IBinaryStream stream)
{
+ // ReSharper disable once InconsistentlySynchronizedField
ICacheEntryEvent<TK, TV> evt = CQU.ReadEvent<TK, TV>(stream, _ignite.Marshaller, _keepBinary);
return _filter.Evaluate(evt);
@@ -91,6 +92,7 @@ namespace Apache.Ignite.Core.Impl.Cache.Query.Continuous
/** <inheritDoc /> */
public void Inject(Ignite grid)
{
+ // ReSharper disable once InconsistentlySynchronizedField
_ignite = grid;
ResourceProcessor.Inject(_filter, grid);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteArgumentCheck.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteArgumentCheck.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteArgumentCheck.cs
index a633291..967225b 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteArgumentCheck.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteArgumentCheck.cs
@@ -15,6 +15,7 @@
* limitations under the License.
*/
+// ReSharper disable ParameterOnlyUsedForPreconditionCheck.Global
namespace Apache.Ignite.Core.Impl.Common
{
using System;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Handle/HandleRegistry.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Handle/HandleRegistry.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Handle/HandleRegistry.cs
index fee7d79..d3d4b30 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Handle/HandleRegistry.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Handle/HandleRegistry.cs
@@ -239,6 +239,7 @@ namespace Apache.Ignite.Core.Impl.Handle
/// <param name="id">Identifier.</param>
/// <param name="throwOnAbsent">Whether to throw an exception if resource is not found.</param>
/// <returns>Target.</returns>
+ // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Global
public T Get<T>(long id, bool throwOnAbsent)
{
object target;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs
index 6d706e8..5c781b5 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs
@@ -145,6 +145,7 @@ namespace Apache.Ignite.Core.Impl.Unmanaged.Jni
/// <summary>
/// Gets the JVM.
/// </summary>
+ // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Global
public static Jvm Get(bool ignoreMissing = false)
{
var res = _instance;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Core/Ssl/SslContextFactory.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Ssl/SslContextFactory.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Ssl/SslContextFactory.cs
index bb5c7a0..a55c6f1 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Ssl/SslContextFactory.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Ssl/SslContextFactory.cs
@@ -48,10 +48,14 @@ namespace Apache.Ignite.Core.Ssl
/// </summary>
public string KeyStoreType { get; set; }
- /// <inheritdoc />
+ /// <summary>
+ /// Key store file path.
+ /// </summary>
public string KeyStoreFilePath { get; set; }
- /// <inheritdoc />
+ /// <summary>
+ /// Key store file password.
+ /// </summary>
public string KeyStorePassword { get; set; }
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/a69e1acb/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryProvider.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryProvider.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryProvider.cs
index 4f35a42..cfc30cc 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryProvider.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryProvider.cs
@@ -220,17 +220,20 @@ namespace Apache.Ignite.Linq.Impl
// Try with full type name (this works when TableName is not set).
var valueTypeName = cacheValueType.FullName;
- if (validTableNames.Contains(valueTypeName, StringComparer.OrdinalIgnoreCase))
+ if (valueTypeName != null)
{
- return EscapeTableName(valueTypeName);
- }
-
- // Remove namespace and nested class qualification and try again.
- valueTypeName = EscapeTableName(valueTypeName);
-
- if (validTableNames.Contains(valueTypeName, StringComparer.OrdinalIgnoreCase))
- {
- return valueTypeName;
+ if (validTableNames.Contains(valueTypeName, StringComparer.OrdinalIgnoreCase))
+ {
+ return EscapeTableName(valueTypeName);
+ }
+
+ // Remove namespace and nested class qualification and try again.
+ valueTypeName = EscapeTableName(valueTypeName);
+
+ if (validTableNames.Contains(valueTypeName, StringComparer.OrdinalIgnoreCase))
+ {
+ return valueTypeName;
+ }
}
throw new CacheException(string.Format("Table name cannot be inferred for cache '{0}', " +
[39/50] [abbrv] ignite git commit: IGNITE-7929 Web Console: Fixed
popover width.
Posted by sb...@apache.org.
IGNITE-7929 Web Console: Fixed popover width.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5b402434
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5b402434
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5b402434
Branch: refs/heads/ignite-zk
Commit: 5b4024342889d396ae19c2224c85c2da20d83e2f
Parents: 6d85177
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Fri Mar 16 17:13:41 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Mar 16 17:13:41 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/public/stylesheets/style.scss | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b402434/modules/web-console/frontend/public/stylesheets/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index ece3b65..fe1e94e 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -1183,7 +1183,9 @@ button.form-control {
}
.theme-line .popover.validation-error {
- max-width: 400px;
+ white-space: pre-wrap;
+ width: auto !important;
+ max-width: 400px !important;
color: $brand-primary;
background: white;
border: 1px solid $brand-primary;
[30/50] [abbrv] ignite git commit: IGNITE-7898 Fixed
IgniteCachePartitionLossPolicy tests - Fixes #3626.
Posted by sb...@apache.org.
IGNITE-7898 Fixed IgniteCachePartitionLossPolicy tests - Fixes #3626.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e7ee88aa
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e7ee88aa
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e7ee88aa
Branch: refs/heads/ignite-zk
Commit: e7ee88aa82b3c59eb1ae671e2d33f1307f17162e
Parents: 798c01b
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Wed Mar 14 16:51:54 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Mar 14 16:51:54 2018 +0300
----------------------------------------------------------------------
.../IgniteCachePartitionLossPolicySelfTest.java | 29 ++++++++++++++++----
1 file changed, 24 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e7ee88aa/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
index 23d3941..7f35ddb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
@@ -17,11 +17,6 @@
package org.apache.ignite.internal.processors.cache.distributed;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import javax.cache.CacheException;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.PartitionLossPolicy;
@@ -34,14 +29,22 @@ import org.apache.ignite.events.CacheRebalancingEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.events.EventType;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.P1;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import javax.cache.CacheException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
@@ -158,6 +161,22 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe
int part = prepareTopology();
+ // Wait for all grids (servers and client) have same topology version
+ // to make sure that all nodes received map with lost partition.
+ GridTestUtils.waitForCondition(() -> {
+ AffinityTopologyVersion last = null;
+ for (Ignite ig : G.allGrids()) {
+ AffinityTopologyVersion ver = ((IgniteEx) ig).context().cache().context().exchange().readyAffinityVersion();
+
+ if (last != null && !last.equals(ver))
+ return false;
+
+ last = ver;
+ }
+
+ return true;
+ }, 10000);
+
for (Ignite ig : G.allGrids()) {
info("Checking node: " + ig.cluster().localNode().id());
[08/50] [abbrv] ignite git commit: IGNITE-7897: Add example for LSQR
with data normalization.
Posted by sb...@apache.org.
IGNITE-7897: Add example for LSQR with data normalization.
This closes #3614
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6dbcf7e3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6dbcf7e3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6dbcf7e3
Branch: refs/heads/ignite-zk
Commit: 6dbcf7e34ec3b1d3cd80fc83ab53a9959c3b1fe1
Parents: a8170f7
Author: Anton Dmitriev <dm...@gmail.com>
Authored: Wed Mar 7 15:11:53 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Wed Mar 7 15:11:53 2018 +0300
----------------------------------------------------------------------
...nWithLSQRTrainerAndNormalizationExample.java | 184 +++++++++++++++++++
1 file changed, 184 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6dbcf7e3/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
new file mode 100644
index 0000000..61195c4
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.regression.linear;
+
+import java.util.Arrays;
+import java.util.UUID;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.examples.ml.math.matrix.SparseDistributedMatrixExample;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.preprocessing.normalization.NormalizationPreprocessor;
+import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
+import org.apache.ignite.ml.regressions.linear.LinearRegressionLSQRTrainer;
+import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * Run linear regression model over distributed matrix.
+ *
+ * @see LinearRegressionLSQRTrainer
+ * @see NormalizationTrainer
+ * @see NormalizationPreprocessor
+ */
+public class DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample {
+ /** */
+ private static final double[][] data = {
+ {8, 78, 284, 9.100000381, 109},
+ {9.300000191, 68, 433, 8.699999809, 144},
+ {7.5, 70, 739, 7.199999809, 113},
+ {8.899999619, 96, 1792, 8.899999619, 97},
+ {10.19999981, 74, 477, 8.300000191, 206},
+ {8.300000191, 111, 362, 10.89999962, 124},
+ {8.800000191, 77, 671, 10, 152},
+ {8.800000191, 168, 636, 9.100000381, 162},
+ {10.69999981, 82, 329, 8.699999809, 150},
+ {11.69999981, 89, 634, 7.599999905, 134},
+ {8.5, 149, 631, 10.80000019, 292},
+ {8.300000191, 60, 257, 9.5, 108},
+ {8.199999809, 96, 284, 8.800000191, 111},
+ {7.900000095, 83, 603, 9.5, 182},
+ {10.30000019, 130, 686, 8.699999809, 129},
+ {7.400000095, 145, 345, 11.19999981, 158},
+ {9.600000381, 112, 1357, 9.699999809, 186},
+ {9.300000191, 131, 544, 9.600000381, 177},
+ {10.60000038, 80, 205, 9.100000381, 127},
+ {9.699999809, 130, 1264, 9.199999809, 179},
+ {11.60000038, 140, 688, 8.300000191, 80},
+ {8.100000381, 154, 354, 8.399999619, 103},
+ {9.800000191, 118, 1632, 9.399999619, 101},
+ {7.400000095, 94, 348, 9.800000191, 117},
+ {9.399999619, 119, 370, 10.39999962, 88},
+ {11.19999981, 153, 648, 9.899999619, 78},
+ {9.100000381, 116, 366, 9.199999809, 102},
+ {10.5, 97, 540, 10.30000019, 95},
+ {11.89999962, 176, 680, 8.899999619, 80},
+ {8.399999619, 75, 345, 9.600000381, 92},
+ {5, 134, 525, 10.30000019, 126},
+ {9.800000191, 161, 870, 10.39999962, 108},
+ {9.800000191, 111, 669, 9.699999809, 77},
+ {10.80000019, 114, 452, 9.600000381, 60},
+ {10.10000038, 142, 430, 10.69999981, 71},
+ {10.89999962, 238, 822, 10.30000019, 86},
+ {9.199999809, 78, 190, 10.69999981, 93},
+ {8.300000191, 196, 867, 9.600000381, 106},
+ {7.300000191, 125, 969, 10.5, 162},
+ {9.399999619, 82, 499, 7.699999809, 95},
+ {9.399999619, 125, 925, 10.19999981, 91},
+ {9.800000191, 129, 353, 9.899999619, 52},
+ {3.599999905, 84, 288, 8.399999619, 110},
+ {8.399999619, 183, 718, 10.39999962, 69},
+ {10.80000019, 119, 540, 9.199999809, 57},
+ {10.10000038, 180, 668, 13, 106},
+ {9, 82, 347, 8.800000191, 40},
+ {10, 71, 345, 9.199999809, 50},
+ {11.30000019, 118, 463, 7.800000191, 35},
+ {11.30000019, 121, 728, 8.199999809, 86},
+ {12.80000019, 68, 383, 7.400000095, 57},
+ {10, 112, 316, 10.39999962, 57},
+ {6.699999809, 109, 388, 8.899999619, 94}
+ };
+
+ /** Run example. */
+ public static void main(String[] args) throws InterruptedException {
+ System.out.println();
+ System.out.println(">>> Linear regression model over sparse distributed matrix API usage example started.");
+ // Start ignite grid.
+ try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+ System.out.println(">>> Ignite grid started.");
+
+ // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
+ // because we create ignite cache internally.
+ IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+ SparseDistributedMatrixExample.class.getSimpleName(), () -> {
+ IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
+
+ System.out.println(">>> Create new normalization trainer object.");
+ NormalizationTrainer<Integer, double[]> normalizationTrainer = new NormalizationTrainer<>();
+
+ System.out.println(">>> Perform the training to get the normalization preprocessor.");
+ NormalizationPreprocessor<Integer, double[]> preprocessor = normalizationTrainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, dataCache),
+ (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+ 4
+ );
+
+ System.out.println(">>> Create new linear regression trainer object.");
+ LinearRegressionLSQRTrainer<Integer, double[]> trainer = new LinearRegressionLSQRTrainer<>();
+
+ System.out.println(">>> Perform the training to get the model.");
+ LinearRegressionModel mdl = trainer.fit(
+ new CacheBasedDatasetBuilder<>(ignite, dataCache),
+ preprocessor,
+ (k, v) -> v[0],
+ 4
+ );
+
+ System.out.println(">>> Linear regression model: " + mdl);
+
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
+
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ Integer key = observation.getKey();
+ double[] val = observation.getValue();
+ double groundTruth = val[0];
+
+ double prediction = mdl.apply(new DenseLocalOnHeapVector(preprocessor.apply(key, val)));
+
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
+ }
+ }
+
+ System.out.println(">>> ---------------------------------");
+ });
+
+ igniteThread.start();
+
+ igniteThread.join();
+ }
+ }
+
+ /**
+ * Fills cache with data and returns it.
+ *
+ * @param ignite Ignite instance.
+ * @return Filled Ignite Cache.
+ */
+ private static IgniteCache<Integer, double[]> getTestCache(Ignite ignite) {
+ CacheConfiguration<Integer, double[]> cacheConfiguration = new CacheConfiguration<>();
+ cacheConfiguration.setName("TEST_" + UUID.randomUUID());
+ cacheConfiguration.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+ IgniteCache<Integer, double[]> cache = ignite.createCache(cacheConfiguration);
+
+ for (int i = 0; i < data.length; i++)
+ cache.put(i, data[i]);
+
+ return cache;
+ }
+}
[40/50] [abbrv] ignite git commit: IGNITE-7970 Web Console: Fixed
typo in notification test.
Posted by sb...@apache.org.
IGNITE-7970 Web Console: Fixed typo in notification test.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/094f6c32
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/094f6c32
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/094f6c32
Branch: refs/heads/ignite-zk
Commit: 094f6c328b8297cb55335dea6bec391308bbed27
Parents: 5b40243
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Fri Mar 16 17:25:59 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Mar 16 17:25:59 2018 +0700
----------------------------------------------------------------------
modules/web-console/e2e/testcafe/fixtures/admin-panel.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/094f6c32/modules/web-console/e2e/testcafe/fixtures/admin-panel.js
----------------------------------------------------------------------
diff --git a/modules/web-console/e2e/testcafe/fixtures/admin-panel.js b/modules/web-console/e2e/testcafe/fixtures/admin-panel.js
index 9baf055..5400d79 100644
--- a/modules/web-console/e2e/testcafe/fixtures/admin-panel.js
+++ b/modules/web-console/e2e/testcafe/fixtures/admin-panel.js
@@ -36,7 +36,7 @@ fixture('Checking admin panel')
});
test('Testing setting notifications', async(t) => {
- await t.click(Selector('button').withAttribute('ng-click', 'ctrl.changeUserNotifications()'));
+ await t.click(Selector('button').withAttribute('ng-click', '$ctrl.changeUserNotifications()'));
await t
.expect(Selector('h4').withText(/.*Set user notifications.*/).exists)
@@ -50,7 +50,7 @@ test('Testing setting notifications', async(t) => {
.expect(Selector('.wch-notification').innerText)
.eql('test message');
- await t.click(Selector('button').withAttribute('ng-click', 'ctrl.changeUserNotifications()'));
+ await t.click(Selector('button').withAttribute('ng-click', '$ctrl.changeUserNotifications()'));
await t
.click('.ace_content')
[31/50] [abbrv] ignite git commit: IGNITE-7756: IgniteUuid added to
predefined types - Fixes #3550.
Posted by sb...@apache.org.
IGNITE-7756: IgniteUuid added to predefined types - Fixes #3550.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/70ca86a3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/70ca86a3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/70ca86a3
Branch: refs/heads/ignite-zk
Commit: 70ca86a30a7589f9ff466b93a958362135347d02
Parents: e7ee88a
Author: Nikolay Izhikov <ni...@apache.org>
Authored: Wed Mar 14 22:33:47 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Wed Mar 14 22:34:32 2018 +0300
----------------------------------------------------------------------
.../IgniteUuidCompatibilityTest.java | 189 +++++++++++++++++++
.../IgniteCompatibilityBasicTestSuite.java | 3 +
.../ignite/internal/binary/BinaryContext.java | 1 +
.../client/IgniteDataStreamerTest.java | 118 ++++++++++++
.../IgniteCacheWithIndexingTestSuite.java | 3 +
5 files changed, 314 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/70ca86a3/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgniteUuidCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgniteUuidCompatibilityTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgniteUuidCompatibilityTest.java
new file mode 100644
index 0000000..88c6900
--- /dev/null
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgniteUuidCompatibilityTest.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.compatibility.persistence;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.binary.BinaryContext;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+
+/**
+ * Checks that index and caches with IgniteUuid are compatible when IgniteUuid becomes predefined type.
+ *
+ * @see BinaryContext#registerPredefinedType(Class, int)
+ */
+public class IgniteUuidCompatibilityTest extends IgnitePersistenceCompatibilityAbstractTest {
+ /** */
+ protected static final String TEST_CACHE_NAME = IgniteUuidCompatibilityTest.class.getSimpleName();
+
+ /** */
+ protected volatile boolean compactFooter;
+
+ /** */
+ private static Set<String> VALUES = new HashSet<>(Arrays.asList("one", "two", "three"));
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "binary_meta", false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setPeerClassLoadingEnabled(false);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ ));
+
+ cfg.setBinaryConfiguration(
+ new BinaryConfiguration()
+ .setCompactFooter(compactFooter)
+ );
+
+ return cfg;
+ }
+
+ /**
+ * Tests opportunity to read data from previous Ignite DB version for cache contains IgniteUuid instances.
+ *
+ * @throws Exception If failed.
+ */
+ public void testIgniteUuid_2_3() throws Exception {
+ doTestIgniteUuidCompatibility("2.3.0", false);
+ }
+
+ /**
+ * Tests opportunity to read data from previous Ignite DB version for cache contains IgniteUuid instances.
+ *
+ * @throws Exception If failed.
+ */
+ public void testIgniteUuidWithCompactFooter_2_3() throws Exception {
+ doTestIgniteUuidCompatibility("2.3.0", true);
+ }
+
+ /**
+ * Tests opportunity to read data from previous Ignite DB version.
+ *
+ * @param igniteVer 3-digits version of ignite
+ * @throws Exception If failed.
+ */
+ private void doTestIgniteUuidCompatibility(String igniteVer, boolean compactFooter) throws Exception {
+ boolean prev = this.compactFooter;
+
+ try {
+ this.compactFooter = compactFooter;
+
+ startGrid(1, igniteVer, new ConfigurationClosure(compactFooter), new PutData());
+
+ stopAllGrids();
+
+ IgniteEx ignite = startGrid(0);
+
+ assertEquals(1, ignite.context().discovery().topologyVersion());
+
+ ignite.active(true);
+
+ IgniteCache<IgniteUuid, String> cache = ignite.cache(TEST_CACHE_NAME);
+
+ assertEquals(cache.size(CachePeekMode.ALL), VALUES.size());
+
+ Set<String> values = new HashSet<>();
+
+ for (Cache.Entry<IgniteUuid, String> e : cache)
+ values.add(e.getValue());
+
+ assertEquals(values, VALUES);
+ }
+ finally {
+ stopAllGrids();
+
+ this.compactFooter = prev;
+ }
+ }
+
+ /** */
+ public static class ConfigurationClosure implements IgniteInClosure<IgniteConfiguration> {
+ private boolean compactFooter;
+
+ public ConfigurationClosure(boolean compactFooter) {
+ this.compactFooter = compactFooter;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void apply(IgniteConfiguration cfg) {
+ cfg.setLocalHost("127.0.0.1");
+
+ TcpDiscoverySpi disco = new TcpDiscoverySpi();
+ disco.setIpFinder(GridCacheAbstractFullApiSelfTest.LOCAL_IP_FINDER);
+
+ cfg.setDiscoverySpi(disco);
+
+ cfg.setPeerClassLoadingEnabled(false);
+
+ cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
+
+ if (!compactFooter)
+ cfg.setBinaryConfiguration(new BinaryConfiguration().setCompactFooter(compactFooter));
+ }
+ }
+
+ /** */
+ public static class PutData implements IgniteInClosure<Ignite> {
+ /** {@inheritDoc} */
+ @Override public void apply(Ignite ignite) {
+ ignite.active(true);
+
+ CacheConfiguration<IgniteUuid, String> cacheCfg = new CacheConfiguration<>();
+
+ cacheCfg.setName(TEST_CACHE_NAME);
+ cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ cacheCfg.setBackups(1);
+ cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+
+ IgniteCache<IgniteUuid, String> cache = ignite.createCache(cacheCfg);
+
+ for (String v : VALUES)
+ cache.put(IgniteUuid.randomUuid(), v);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/70ca86a3/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
index 20643d4..eaa38af 100644
--- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
@@ -20,6 +20,7 @@ package org.apache.ignite.compatibility.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.compatibility.persistence.DummyPersistenceCompatibilityTest;
import org.apache.ignite.compatibility.persistence.FoldersReuseCompatibilityTest;
+import org.apache.ignite.compatibility.persistence.IgniteUuidCompatibilityTest;
import org.apache.ignite.compatibility.persistence.MigratingToWalV2SerializerWithCompactionTest;
/**
@@ -39,6 +40,8 @@ public class IgniteCompatibilityBasicTestSuite {
suite.addTestSuite(MigratingToWalV2SerializerWithCompactionTest.class);
+ suite.addTestSuite(IgniteUuidCompatibilityTest.class);
+
return suite;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/70ca86a3/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index 9606139..8f85947 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -333,6 +333,7 @@ public class BinaryContext {
registerPredefinedType(GridMapEntry.class, 60);
registerPredefinedType(IgniteBiTuple.class, 61);
registerPredefinedType(T2.class, 62);
+ registerPredefinedType(IgniteUuid.class, 63);
registerPredefinedType(PlatformJavaObjectFactoryProxy.class,
GridBinaryMarshaller.PLATFORM_JAVA_OBJECT_FACTORY_PROXY);
http://git-wip-us.apache.org/repos/asf/ignite/blob/70ca86a3/modules/indexing/src/test/java/org/apache/ignite/internal/processors/client/IgniteDataStreamerTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/client/IgniteDataStreamerTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/client/IgniteDataStreamerTest.java
new file mode 100644
index 0000000..1bfb02e
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/client/IgniteDataStreamerTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.client;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CachePeekMode.ALL;
+
+/**
+ */
+public class IgniteDataStreamerTest extends GridCommonAbstractTest {
+ public static final String CACHE_NAME = "UUID_CACHE";
+
+ public static final int DATA_SIZE = 3;
+
+ public static final long WAIT_TIMEOUT = 30_000L;
+
+ private boolean client = false;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ if (client)
+ cfg.setClientMode(true);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ startGrids(2);
+
+ client = true;
+
+ startGrid("client");
+ }
+
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ grid("client").destroyCache(CACHE_NAME);
+ }
+
+ /**
+ * @return Cache configuration
+ */
+ private <K, V> CacheConfiguration<K, V> cacheConfiguration(Class<K> key, Class<V> value) {
+ CacheConfiguration<K, V> ccfg = new CacheConfiguration<>(CACHE_NAME);
+
+ ccfg.setIndexedTypes(key, value);
+
+ return ccfg;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testStreamerIgniteUuid() throws Exception {
+ Ignite client = grid("client");
+
+ IgniteCache<IgniteUuid, Integer> cache =
+ client.createCache(cacheConfiguration(IgniteUuid.class, Integer.class));
+
+ try(IgniteDataStreamer<IgniteUuid, Integer> streamer = client.dataStreamer(CACHE_NAME)) {
+ assertTrue("Expecting " + DataStreamerImpl.class.getName(), streamer instanceof DataStreamerImpl);
+
+ ((DataStreamerImpl<IgniteUuid, Integer>)streamer).maxRemapCount(0);
+
+ List<IgniteFuture> futs = new ArrayList<>();
+
+ for(int i=0; i<DATA_SIZE; i++) {
+ IgniteFuture<?> fut = streamer.addData(IgniteUuid.randomUuid(), i);
+
+ futs.add(fut);
+ }
+
+ streamer.flush();
+
+ for (IgniteFuture fut : futs) {
+ //This should not throw any exception.
+ Object res = fut.get(WAIT_TIMEOUT);
+
+ //Printing future result to log to prevent jvm optimization
+ log.debug(res == null ? null : res.toString());
+ }
+
+ assertTrue(cache.size(ALL) == DATA_SIZE);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/70ca86a3/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
index 4047700..bc99981 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
@@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.ttl.CacheTtlAtomicLocalSelfTe
import org.apache.ignite.internal.processors.cache.ttl.CacheTtlAtomicPartitionedSelfTest;
import org.apache.ignite.internal.processors.cache.ttl.CacheTtlTransactionalLocalSelfTest;
import org.apache.ignite.internal.processors.cache.ttl.CacheTtlTransactionalPartitionedSelfTest;
+import org.apache.ignite.internal.processors.client.IgniteDataStreamerTest;
import org.apache.ignite.internal.processors.query.h2.database.InlineIndexHelperTest;
/**
@@ -78,6 +79,8 @@ public class IgniteCacheWithIndexingTestSuite extends TestSuite {
suite.addTestSuite(IgniteCacheGroupsSqlTest.class);
+ suite.addTestSuite(IgniteDataStreamerTest.class);
+
return suite;
}
}
[28/50] [abbrv] ignite git commit: IGNITE-7946 Fail test with known
issue - Fixes #3631.
Posted by sb...@apache.org.
IGNITE-7946 Fail test with known issue - Fixes #3631.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/48a66237
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/48a66237
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/48a66237
Branch: refs/heads/ignite-zk
Commit: 48a66237c1fb2b5e095eed0c0961764b897230f8
Parents: c533e2c
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Wed Mar 14 13:58:20 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Mar 14 13:58:20 2018 +0300
----------------------------------------------------------------------
.../near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/48a66237/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
index 996d6a7..de7ee5f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
@@ -209,6 +209,7 @@ public class IgniteCacheClientQueryReplicatedNodeRestartSelfTest extends GridCom
* @throws Exception If failed.
*/
public void testRestarts() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-7946");
int duration = 90 * 1000;
int qryThreadNum = 5;
int restartThreadsNum = 2; // 2 of 4 data nodes
[26/50] [abbrv] ignite git commit: IGNITE-7253: JDBC thin driver:
implemented streaming. This closes #3499. This closes #3591.
Posted by sb...@apache.org.
IGNITE-7253: JDBC thin driver: implemented streaming. This closes #3499. This closes #3591.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7366809e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7366809e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7366809e
Branch: refs/heads/ignite-zk
Commit: 7366809ed0ea7a943e1c80079fdbeb556ee86245
Parents: ae2bf3d
Author: Alexander Paschenko <al...@gmail.com>
Authored: Wed Mar 14 12:23:37 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Mar 14 12:23:37 2018 +0300
----------------------------------------------------------------------
.../jdbc2/JdbcNoCacheStreamingSelfTest.java | 182 +++++++
.../internal/jdbc2/JdbcStreamingSelfTest.java | 220 +++++++--
.../jdbc/suite/IgniteJdbcDriverTestSuite.java | 11 +-
.../jdbc/thin/JdbcThinStreamingSelfTest.java | 486 +++++++++++++++++++
.../internal/jdbc/thin/JdbcThinConnection.java | 112 +++++
.../jdbc/thin/JdbcThinPreparedStatement.java | 22 +-
.../internal/jdbc/thin/JdbcThinStatement.java | 153 +++++-
.../internal/jdbc/thin/JdbcThinTcpIo.java | 12 +-
.../ignite/internal/jdbc2/JdbcConnection.java | 10 +-
.../jdbc2/JdbcStreamedPreparedStatement.java | 2 +-
.../processors/cache/IgniteCacheProxyImpl.java | 4 +-
.../odbc/jdbc/JdbcBatchExecuteRequest.java | 40 +-
.../odbc/jdbc/JdbcConnectionContext.java | 13 +-
.../odbc/jdbc/JdbcRequestHandler.java | 84 ++--
.../processors/query/GridQueryIndexing.java | 32 +-
.../processors/query/GridQueryProcessor.java | 50 +-
.../processors/query/SqlClientContext.java | 223 +++++++++
.../apache/ignite/internal/sql/SqlKeyword.java | 29 +-
.../apache/ignite/internal/sql/SqlParser.java | 26 +-
.../ignite/internal/sql/SqlParserUtils.java | 22 +
.../sql/command/SqlSetStreamingCommand.java | 191 ++++++++
...IgniteClientCacheInitializationFailTest.java | 15 +-
.../sql/SqlParserSetStreamingSelfTest.java | 134 +++++
.../ignite/testframework/GridTestUtils.java | 2 +-
.../query/h2/DmlStatementsProcessor.java | 104 ++--
.../internal/processors/query/h2/H2Utils.java | 28 +-
.../processors/query/h2/IgniteH2Indexing.java | 134 +++--
.../query/h2/ddl/DdlStatementsProcessor.java | 10 +-
.../processors/query/h2/dml/UpdatePlan.java | 2 +-
.../query/h2/dml/UpdatePlanBuilder.java | 13 +-
.../query/h2/sql/GridSqlQueryParser.java | 12 +
.../cache/GridCacheCrossCacheQuerySelfTest.java | 18 +-
.../IgniteCacheQuerySelfTestSuite.java | 2 +
33 files changed, 2140 insertions(+), 258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcNoCacheStreamingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcNoCacheStreamingSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcNoCacheStreamingSelfTest.java
new file mode 100644
index 0000000..74c2820
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcNoCacheStreamingSelfTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.jdbc2;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.util.Collections;
+import java.util.Properties;
+import org.apache.ignite.IgniteJdbcDriver;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteJdbcDriver.CFG_URL_PREFIX;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ * Data streaming test for thick driver and no explicit caches.
+ */
+public class JdbcNoCacheStreamingSelfTest extends GridCommonAbstractTest {
+ /** JDBC URL. */
+ private static final String BASE_URL = CFG_URL_PREFIX +
+ "cache=default@modules/clients/src/test/config/jdbc-config.xml";
+
+ /** Connection. */
+ protected Connection conn;
+
+ /** */
+ protected transient IgniteLogger log;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ return getConfiguration0(gridName);
+ }
+
+ /**
+ * @param gridName Grid name.
+ * @return Grid configuration used for starting the grid.
+ * @throws Exception If failed.
+ */
+ private IgniteConfiguration getConfiguration0(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration<?,?> cache = defaultCacheConfiguration();
+
+ cache.setCacheMode(PARTITIONED);
+ cache.setBackups(1);
+ cache.setWriteSynchronizationMode(FULL_SYNC);
+ cache.setIndexedTypes(
+ Integer.class, Integer.class
+ );
+
+ cfg.setCacheConfiguration(cache);
+ cfg.setLocalHost("127.0.0.1");
+
+ TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+ TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+ ipFinder.setAddresses(Collections.singleton("127.0.0.1:47500..47501"));
+
+ disco.setIpFinder(ipFinder);
+
+ cfg.setDiscoverySpi(disco);
+
+ cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ startGrids(2);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ * @param allowOverwrite Allow overwriting of existing keys.
+ * @return Connection to use for the test.
+ * @throws Exception if failed.
+ */
+ protected Connection createConnection(boolean allowOverwrite) throws Exception {
+ Properties props = new Properties();
+
+ props.setProperty(IgniteJdbcDriver.PROP_STREAMING, "true");
+ props.setProperty(IgniteJdbcDriver.PROP_STREAMING_FLUSH_FREQ, "500");
+
+ if (allowOverwrite)
+ props.setProperty(IgniteJdbcDriver.PROP_STREAMING_ALLOW_OVERWRITE, "true");
+
+ return DriverManager.getConnection(BASE_URL, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ U.closeQuiet(conn);
+
+ ignite(0).cache(DEFAULT_CACHE_NAME).clear();
+
+ super.afterTest();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testStreamedInsert() throws Exception {
+ for (int i = 10; i <= 100; i += 10)
+ ignite(0).cache(DEFAULT_CACHE_NAME).put(i, i * 100);
+
+ try (Connection conn = createConnection(false)) {
+ try (PreparedStatement stmt = conn.prepareStatement("insert into Integer(_key, _val) values (?, ?)")) {
+ for (int i = 1; i <= 100; i++) {
+ stmt.setInt(1, i);
+ stmt.setInt(2, i);
+
+ stmt.executeUpdate();
+ }
+ }
+ }
+
+ U.sleep(500);
+
+ // Now let's check it's all there.
+ for (int i = 1; i <= 100; i++) {
+ if (i % 10 != 0)
+ assertEquals(i, grid(0).cache(DEFAULT_CACHE_NAME).get(i));
+ else // All that divides by 10 evenly should point to numbers 100 times greater - see above
+ assertEquals(i * 100, grid(0).cache(DEFAULT_CACHE_NAME).get(i));
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testStreamedInsertWithOverwritesAllowed() throws Exception {
+ for (int i = 10; i <= 100; i += 10)
+ ignite(0).cache(DEFAULT_CACHE_NAME).put(i, i * 100);
+
+ try (Connection conn = createConnection(true)) {
+ try (PreparedStatement stmt = conn.prepareStatement("insert into Integer(_key, _val) values (?, ?)")) {
+ for (int i = 1; i <= 100; i++) {
+ stmt.setInt(1, i);
+ stmt.setInt(2, i);
+
+ stmt.executeUpdate();
+ }
+ }
+ }
+
+ U.sleep(500);
+
+ // Now let's check it's all there.
+ // i should point to i at all times as we've turned overwrites on above.
+ for (int i = 1; i <= 100; i++)
+ assertEquals(i, grid(0).cache(DEFAULT_CACHE_NAME).get(i));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
index 5418ca0..10adedc 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
@@ -20,16 +20,24 @@ package org.apache.ignite.internal.jdbc2;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
import java.util.Collections;
import java.util.Properties;
+import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteJdbcDriver;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectBuilder;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.ConnectorConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.query.QueryUtils;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import static org.apache.ignite.IgniteJdbcDriver.CFG_URL_PREFIX;
@@ -41,10 +49,12 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
*/
public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
/** JDBC URL. */
- private static final String BASE_URL = CFG_URL_PREFIX + "cache=default@modules/clients/src/test/config/jdbc-config.xml";
+ private static final String BASE_URL = CFG_URL_PREFIX +
+ "cache=default@modules/clients/src/test/config/jdbc-config.xml";
- /** Connection. */
- protected Connection conn;
+ /** Streaming URL. */
+ private static final String STREAMING_URL = CFG_URL_PREFIX +
+ "cache=person@modules/clients/src/test/config/jdbc-config.xml";
/** */
protected transient IgniteLogger log;
@@ -90,7 +100,18 @@ public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
startGrids(2);
+
+ try (Connection c = createOrdinaryConnection()) {
+ try (Statement s = c.createStatement()) {
+ s.execute("CREATE TABLE PUBLIC.Person(\"id\" int primary key, \"name\" varchar) WITH " +
+ "\"cache_name=person,value_type=Person\"");
+ }
+ }
+
+ U.sleep(1000);
}
/** {@inheritDoc} */
@@ -99,27 +120,51 @@ public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
}
/**
+ * @return Connection without streaming initially turned on.
+ * @throws SQLException if failed.
+ */
+ protected Connection createOrdinaryConnection() throws SQLException {
+ Connection res = DriverManager.getConnection(BASE_URL, new Properties());
+
+ res.setSchema(QueryUtils.DFLT_SCHEMA);
+
+ return res;
+ }
+
+ /**
* @param allowOverwrite Allow overwriting of existing keys.
* @return Connection to use for the test.
* @throws Exception if failed.
*/
- private Connection createConnection(boolean allowOverwrite) throws Exception {
+ protected Connection createStreamedConnection(boolean allowOverwrite) throws Exception {
+ return createStreamedConnection(allowOverwrite, 500);
+ }
+
+ /**
+ * @param allowOverwrite Allow overwriting of existing keys.
+ * @param flushTimeout Stream flush timeout.
+ * @return Connection to use for the test.
+ * @throws Exception if failed.
+ */
+ protected Connection createStreamedConnection(boolean allowOverwrite, long flushTimeout) throws Exception {
Properties props = new Properties();
props.setProperty(IgniteJdbcDriver.PROP_STREAMING, "true");
- props.setProperty(IgniteJdbcDriver.PROP_STREAMING_FLUSH_FREQ, "500");
+ props.setProperty(IgniteJdbcDriver.PROP_STREAMING_FLUSH_FREQ, String.valueOf(flushTimeout));
if (allowOverwrite)
props.setProperty(IgniteJdbcDriver.PROP_STREAMING_ALLOW_OVERWRITE, "true");
- return DriverManager.getConnection(BASE_URL, props);
+ Connection res = DriverManager.getConnection(STREAMING_URL, props);
+
+ res.setSchema(QueryUtils.DFLT_SCHEMA);
+
+ return res;
}
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
- U.closeQuiet(conn);
-
- ignite(0).cache(DEFAULT_CACHE_NAME).clear();
+ cache().clear();
super.afterTest();
}
@@ -128,30 +173,59 @@ public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
* @throws Exception if failed.
*/
public void testStreamedInsert() throws Exception {
- conn = createConnection(false);
-
for (int i = 10; i <= 100; i += 10)
- ignite(0).cache(DEFAULT_CACHE_NAME).put(i, i * 100);
+ put(i, nameForId(i * 100));
+
+ try (Connection conn = createStreamedConnection(false)) {
+ try (PreparedStatement stmt = conn.prepareStatement("insert into PUBLIC.Person(\"id\", \"name\") " +
+ "values (?, ?)")) {
+ for (int i = 1; i <= 100; i++) {
+ stmt.setInt(1, i);
+ stmt.setString(2, nameForId(i));
+
+ stmt.executeUpdate();
+ }
+ }
+ }
- PreparedStatement stmt = conn.prepareStatement("insert into Integer(_key, _val) values (?, ?)");
+ U.sleep(500);
+ // Now let's check it's all there.
for (int i = 1; i <= 100; i++) {
- stmt.setInt(1, i);
- stmt.setInt(2, i);
+ if (i % 10 != 0)
+ assertEquals(nameForId(i), nameForIdInCache(i));
+ else // All that divides by 10 evenly should point to numbers 100 times greater - see above
+ assertEquals(nameForId(i * 100), nameForIdInCache(i));
+ }
+ }
- stmt.executeUpdate();
+ /**
+ * @throws Exception if failed.
+ */
+ public void testStreamedInsertWithoutColumnsList() throws Exception {
+ for (int i = 10; i <= 100; i += 10)
+ put(i, nameForId(i * 100));
+
+ try (Connection conn = createStreamedConnection(false)) {
+ try (PreparedStatement stmt = conn.prepareStatement("insert into PUBLIC.Person(\"id\", \"name\") " +
+ "values (?, ?)")) {
+ for (int i = 1; i <= 100; i++) {
+ stmt.setInt(1, i);
+ stmt.setString(2, nameForId(i));
+
+ stmt.executeUpdate();
+ }
+ }
}
- // Closing connection makes it wait for streamer close
- // and thus for data load completion as well
- conn.close();
+ U.sleep(500);
// Now let's check it's all there.
for (int i = 1; i <= 100; i++) {
if (i % 10 != 0)
- assertEquals(i, grid(0).cache(DEFAULT_CACHE_NAME).get(i));
+ assertEquals(nameForId(i), nameForIdInCache(i));
else // All that divides by 10 evenly should point to numbers 100 times greater - see above
- assertEquals(i * 100, grid(0).cache(DEFAULT_CACHE_NAME).get(i));
+ assertEquals(nameForId(i * 100), nameForIdInCache(i));
}
}
@@ -159,27 +233,101 @@ public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
* @throws Exception if failed.
*/
public void testStreamedInsertWithOverwritesAllowed() throws Exception {
- conn = createConnection(true);
-
for (int i = 10; i <= 100; i += 10)
- ignite(0).cache(DEFAULT_CACHE_NAME).put(i, i * 100);
-
- PreparedStatement stmt = conn.prepareStatement("insert into Integer(_key, _val) values (?, ?)");
-
- for (int i = 1; i <= 100; i++) {
- stmt.setInt(1, i);
- stmt.setInt(2, i);
-
- stmt.executeUpdate();
+ put(i, nameForId(i * 100));
+
+ try (Connection conn = createStreamedConnection(true)) {
+ try (PreparedStatement stmt = conn.prepareStatement("insert into PUBLIC.Person(\"id\", \"name\") " +
+ "values (?, ?)")) {
+ for (int i = 1; i <= 100; i++) {
+ stmt.setInt(1, i);
+ stmt.setString(2, nameForId(i));
+
+ stmt.executeUpdate();
+ }
+ }
}
- // Closing connection makes it wait for streamer close
- // and thus for data load completion as well
- conn.close();
+ U.sleep(500);
// Now let's check it's all there.
// i should point to i at all times as we've turned overwrites on above.
for (int i = 1; i <= 100; i++)
- assertEquals(i, grid(0).cache(DEFAULT_CACHE_NAME).get(i));
+ assertEquals(nameForId(i), nameForIdInCache(i));
+ }
+
+ /** */
+ public void testOnlyInsertsAllowed() {
+ assertStatementForbidden("CREATE TABLE PUBLIC.X (x int primary key, y int)");
+
+ assertStatementForbidden("CREATE INDEX idx_1 ON Person(name)");
+
+ assertStatementForbidden("SELECT * from Person");
+
+ assertStatementForbidden("insert into PUBLIC.Person(\"id\", \"name\") " +
+ "(select \"id\" + 1, CONCAT(\"name\", '1') from Person)");
+
+ assertStatementForbidden("DELETE from Person");
+
+ assertStatementForbidden("UPDATE Person SET \"name\" = 'name0'");
+
+ assertStatementForbidden("alter table Person add column y int");
+ }
+
+ /**
+ * @param sql Statement to check.
+ */
+ @SuppressWarnings("ThrowableNotThrown")
+ protected void assertStatementForbidden(String sql) {
+ GridTestUtils.assertThrows(null, new IgniteCallable<Object>() {
+ @Override public Object call() throws Exception {
+ try (Connection c = createStreamedConnection(false)) {
+ try (PreparedStatement s = c.prepareStatement(sql)) {
+ s.execute();
+ }
+ }
+
+ return null;
+ }
+ }, SQLException.class,"Streaming mode supports only INSERT commands without subqueries.");
+ }
+
+ /**
+ * @return Person cache.
+ */
+ protected IgniteCache<Integer, Object> cache() {
+ return grid(0).cache("person");
+ }
+
+ /**
+ * @param id id of person to put.
+ * @param name name of person to put.
+ */
+ protected void put(int id, String name) {
+ BinaryObjectBuilder bldr = grid(0).binary().builder("Person");
+
+ bldr.setField("name", name);
+
+ cache().put(id, bldr.build());
+ }
+
+ /**
+ * @param id Person id.
+ * @return Default name for person w/given id.
+ */
+ protected String nameForId(int id) {
+ return "Person" + id;
+ }
+
+ /**
+ * @param id person id.
+ * @return Name for person with given id currently stored in cache.
+ */
+ protected String nameForIdInCache(int id) {
+ Object o = cache().withKeepBinary().get(id);
+
+ assertTrue(String.valueOf(o), o instanceof BinaryObject);
+
+ return ((BinaryObject)o).field("name");
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index 4530ae7..f3a94ce 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -42,6 +42,7 @@ import org.apache.ignite.jdbc.thin.JdbcThinBulkLoadTransactionalPartitionedNearS
import org.apache.ignite.jdbc.thin.JdbcThinBulkLoadTransactionalPartitionedSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinBulkLoadTransactionalReplicatedSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinComplexDmlDdlSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinComplexQuerySelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinConnectionSSLTest;
import org.apache.ignite.jdbc.thin.JdbcThinConnectionSelfTest;
@@ -55,8 +56,10 @@ import org.apache.ignite.jdbc.thin.JdbcThinDynamicIndexTransactionalReplicatedSe
import org.apache.ignite.jdbc.thin.JdbcThinEmptyCacheSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinErrorsSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinInsertStatementSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinInsertStatementSkipReducerOnUpdateSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinLocalQueriesSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinMergeStatementSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinMergeStatementSkipReducerOnUpdateSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinMetadataSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinMissingLongArrayResultsTest;
import org.apache.ignite.jdbc.thin.JdbcThinNoDefaultSchemaTest;
@@ -65,11 +68,9 @@ import org.apache.ignite.jdbc.thin.JdbcThinResultSetSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinSchemaCaseTest;
import org.apache.ignite.jdbc.thin.JdbcThinSelectAfterAlterTable;
import org.apache.ignite.jdbc.thin.JdbcThinStatementSelfTest;
-import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSelfTest;
-import org.apache.ignite.jdbc.thin.JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest;
-import org.apache.ignite.jdbc.thin.JdbcThinInsertStatementSkipReducerOnUpdateSelfTest;
-import org.apache.ignite.jdbc.thin.JdbcThinMergeStatementSkipReducerOnUpdateSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinStreamingSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinTcpIoTest;
+import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinWalModeChangeSelfTest;
@@ -121,9 +122,11 @@ public class IgniteJdbcDriverTestSuite extends TestSuite {
suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcStatementBatchingSelfTest.class));
suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcErrorsSelfTest.class));
suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcStreamingToPublicCacheTest.class));
+ suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcNoCacheStreamingSelfTest.class));
suite.addTest(new TestSuite(JdbcBlobTest.class));
suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcStreamingSelfTest.class));
+ suite.addTest(new TestSuite(JdbcThinStreamingSelfTest.class));
// DDL tests.
suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcDynamicIndexAtomicPartitionedNearSelfTest.class));
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinStreamingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinStreamingSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinStreamingSelfTest.java
new file mode 100644
index 0000000..3c36f54
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinStreamingSelfTest.java
@@ -0,0 +1,486 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc.thin;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.internal.jdbc2.JdbcStreamingSelfTest;
+import org.apache.ignite.internal.processors.query.GridQueryCancel;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.SqlClientContext;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Tests for streaming via thin driver.
+ */
+public class JdbcThinStreamingSelfTest extends JdbcStreamingSelfTest {
+ /** */
+ private int batchSize = 17;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ GridQueryProcessor.idxCls = IndexingWithContext.class;
+
+ super.beforeTestsStarted();
+
+ batchSize = 17;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ try (Connection c = createOrdinaryConnection()) {
+ execute(c, "DROP TABLE PUBLIC.T IF EXISTS");
+ }
+
+ IndexingWithContext.cliCtx = null;
+
+ super.afterTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Connection createStreamedConnection(boolean allowOverwrite, long flushFreq) throws Exception {
+ Connection c = JdbcThinAbstractSelfTest.connect(grid(0), null );
+
+ execute(c, "SET STREAMING 1 BATCH_SIZE " + batchSize + " ALLOW_OVERWRITE " + (allowOverwrite ? 1 : 0) +
+ " PER_NODE_BUFFER_SIZE 1000 FLUSH_FREQUENCY " + flushFreq);
+
+ return c;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Connection createOrdinaryConnection() throws SQLException {
+ return JdbcThinAbstractSelfTest.connect(grid(0), null);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testStreamedBatchedInsert() throws Exception {
+ for (int i = 10; i <= 100; i += 10)
+ put(i, nameForId(i * 100));
+
+ try (Connection conn = createStreamedConnection(false)) {
+ assertStreamingState(true);
+
+ try (PreparedStatement stmt = conn.prepareStatement("insert into Person(\"id\", \"name\") values (?, ?), " +
+ "(?, ?)")) {
+ for (int i = 1; i <= 100; i+= 2) {
+ stmt.setInt(1, i);
+ stmt.setString(2, nameForId(i));
+ stmt.setInt(3, i + 1);
+ stmt.setString(4, nameForId(i + 1));
+
+ stmt.addBatch();
+ }
+
+ stmt.executeBatch();
+ }
+ }
+
+ U.sleep(500);
+
+ // Now let's check it's all there.
+ for (int i = 1; i <= 100; i++) {
+ if (i % 10 != 0)
+ assertEquals(nameForId(i), nameForIdInCache(i));
+ else // All that divides by 10 evenly should point to numbers 100 times greater - see above
+ assertEquals(nameForId(i * 100), nameForIdInCache(i));
+ }
+ }
+
+ /**
+ * @throws SQLException if failed.
+ */
+ public void testSimultaneousStreaming() throws Exception {
+ try (Connection anotherConn = createOrdinaryConnection()) {
+ execute(anotherConn, "CREATE TABLE PUBLIC.T(x int primary key, y int) WITH " +
+ "\"cache_name=T,wrap_value=false\"");
+ }
+
+ // Timeout to let connection close be handled on server side.
+ U.sleep(500);
+
+ try (Connection conn = createStreamedConnection(false, 10000)) {
+ assertStreamingState(true);
+
+ PreparedStatement firstStmt = conn.prepareStatement("insert into Person(\"id\", \"name\") values (?, ?)");
+
+ PreparedStatement secondStmt = conn.prepareStatement("insert into PUBLIC.T(x, y) values (?, ?)");
+
+ try {
+ for (int i = 1; i <= 10; i++) {
+ firstStmt.setInt(1, i);
+ firstStmt.setString(2, nameForId(i));
+
+ firstStmt.executeUpdate();
+ }
+
+ for (int i = 51; i <= 67; i++) {
+ secondStmt.setInt(1, i);
+ secondStmt.setInt(2, i);
+
+ secondStmt.executeUpdate();
+ }
+
+ for (int i = 11; i <= 50; i++) {
+ firstStmt.setInt(1, i);
+ firstStmt.setString(2, nameForId(i));
+
+ firstStmt.executeUpdate();
+ }
+
+ for (int i = 68; i <= 100; i++) {
+ secondStmt.setInt(1, i);
+ secondStmt.setInt(2, i);
+
+ secondStmt.executeUpdate();
+ }
+
+ assertCacheEmpty();
+
+ SqlClientContext cliCtx = sqlClientContext();
+
+ HashMap<String, IgniteDataStreamer<?, ?>> streamers = U.field(cliCtx, "streamers");
+
+ assertEquals(2, streamers.size());
+
+ assertEqualsCollections(new HashSet<>(Arrays.asList("person", "T")), streamers.keySet());
+ }
+ finally {
+ U.closeQuiet(firstStmt);
+
+ U.closeQuiet(secondStmt);
+ }
+ }
+
+ // Let's wait a little so that all data arrives to destination - we can't intercept streamers' flush
+ // on connection close in any way.
+ U.sleep(1000);
+
+ // Now let's check it's all there.
+ for (int i = 1; i <= 50; i++)
+ assertEquals(nameForId(i), nameForIdInCache(i));
+
+ for (int i = 51; i <= 100; i++)
+ assertEquals(i, grid(0).cache("T").get(i));
+ }
+
+ /**
+ *
+ */
+ public void testStreamingWithMixedStatementTypes() throws Exception {
+ String prepStmtStr = "insert into Person(\"id\", \"name\") values (?, ?)";
+
+ String stmtStr = "insert into Person(\"id\", \"name\") values (%d, '%s')";
+
+ try (Connection conn = createStreamedConnection(false, 10000)) {
+ assertStreamingState(true);
+
+ PreparedStatement firstStmt = conn.prepareStatement(prepStmtStr);
+
+ Statement secondStmt = conn.createStatement();
+
+ try {
+ for (int i = 1; i <= 100; i++) {
+ boolean usePrep = Math.random() > 0.5;
+
+ boolean useBatch = Math.random() > 0.5;
+
+ if (usePrep) {
+ firstStmt.setInt(1, i);
+ firstStmt.setString(2, nameForId(i));
+
+ if (useBatch)
+ firstStmt.addBatch();
+ else
+ firstStmt.execute();
+ }
+ else {
+ String sql = String.format(stmtStr, i, nameForId(i));
+
+ if (useBatch)
+ secondStmt.addBatch(sql);
+ else
+ secondStmt.execute(sql);
+ }
+ }
+ }
+ finally {
+ U.closeQuiet(firstStmt);
+
+ U.closeQuiet(secondStmt);
+ }
+ }
+
+ // Let's wait a little so that all data arrives to destination - we can't intercept streamers' flush
+ // on connection close in any way.
+ U.sleep(1000);
+
+ // Now let's check it's all there.
+ for (int i = 1; i <= 100; i++)
+ assertEquals(nameForId(i), nameForIdInCache(i));
+ }
+
+ /**
+ * @throws SQLException if failed.
+ */
+ public void testStreamingOffToOn() throws SQLException {
+ try (Connection conn = createOrdinaryConnection()) {
+ assertStreamingState(false);
+
+ execute(conn, "SET STREAMING 1");
+
+ assertStreamingState(true);
+ }
+ }
+
+ /**
+ * @throws SQLException if failed.
+ */
+ public void testStreamingOnToOff() throws Exception {
+ try (Connection conn = createStreamedConnection(false)) {
+ assertStreamingState(true);
+
+ execute(conn, "SET STREAMING off");
+
+ assertStreamingState(false);
+ }
+ }
+
+ /**
+ * @throws SQLException if failed.
+ */
+ public void testFlush() throws Exception {
+ try (Connection conn = createStreamedConnection(false, 10000)) {
+ assertStreamingState(true);
+
+ try (PreparedStatement stmt = conn.prepareStatement("insert into Person(\"id\", \"name\") values (?, ?)")) {
+ for (int i = 1; i <= 100; i++) {
+ stmt.setInt(1, i);
+ stmt.setString(2, nameForId(i));
+
+ stmt.executeUpdate();
+ }
+ }
+
+ assertCacheEmpty();
+
+ execute(conn, "set streaming 0");
+
+ assertStreamingState(false);
+
+ U.sleep(500);
+
+ // Now let's check it's all there.
+ for (int i = 1; i <= 100; i++)
+ assertEquals(nameForId(i), nameForIdInCache(i));
+ }
+ }
+
+ /**
+ * @throws SQLException if failed.
+ */
+ public void testStreamingReEnabled() throws Exception {
+ try (Connection conn = createStreamedConnection(false, 10000)) {
+ assertStreamingState(true);
+
+ try (PreparedStatement stmt = conn.prepareStatement("insert into Person(\"id\", \"name\") values (?, ?)")) {
+ for (int i = 1; i <= 100; i++) {
+ stmt.setInt(1, i);
+ stmt.setString(2, nameForId(i));
+
+ stmt.executeUpdate();
+ }
+ }
+
+ assertCacheEmpty();
+
+ execute(conn, "set streaming 1 batch_size 111 allow_overwrite 0 per_node_buffer_size 512 " +
+ "per_node_parallel_operations 4 flush_frequency 5000");
+
+ U.sleep(500);
+
+ assertEquals((Integer)111, U.field(conn, "streamBatchSize"));
+
+ SqlClientContext cliCtx = sqlClientContext();
+
+ assertTrue(cliCtx.isStream());
+
+ assertFalse(U.field(cliCtx, "streamAllowOverwrite"));
+
+ assertEquals((Integer)512, U.field(cliCtx, "streamNodeBufSize"));
+
+ assertEquals((Long)5000L, U.field(cliCtx, "streamFlushTimeout"));
+
+ assertEquals((Integer)4, U.field(cliCtx, "streamNodeParOps"));
+
+ // Now let's check it's all there - SET STREAMING 1 repeated call must also have caused flush.
+ for (int i = 1; i <= 100; i++)
+ assertEquals(nameForId(i), nameForIdInCache(i));
+ }
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableNotThrown")
+ public void testNonStreamedBatch() {
+ GridTestUtils.assertThrows(null, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ try (Connection conn = createOrdinaryConnection()) {
+ try (Statement s = conn.createStatement()) {
+ for (int i = 1; i <= 10; i++)
+ s.addBatch(String.format("insert into Person(\"id\", \"name\")values (%d, '%s')", i,
+ nameForId(i)));
+
+ execute(conn, "SET STREAMING 1");
+
+ s.addBatch(String.format("insert into Person(\"id\", \"name\")values (%d, '%s')", 11,
+ nameForId(11)));
+ }
+ }
+
+ return null;
+ }
+ }, SQLException.class, "Statement has non-empty batch (call executeBatch() or clearBatch() before " +
+ "enabling streaming).");
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableNotThrown")
+ public void testStreamingStatementInTheMiddleOfNonPreparedBatch() {
+ GridTestUtils.assertThrows(null, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ try (Connection conn = createOrdinaryConnection()) {
+ try (Statement s = conn.createStatement()) {
+ s.addBatch(String.format("insert into Person(\"id\", \"name\")values (%d, '%s')", 1,
+ nameForId(1)));
+
+ s.addBatch("SET STREAMING 1 FLUSH_FREQUENCY 10000");
+ }
+ }
+
+ return null;
+ }
+ }, SQLException.class, "Streaming control commands must be executed explicitly");
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("ThrowableNotThrown")
+ public void testBatchingSetStreamingStatement() {
+ GridTestUtils.assertThrows(null, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ try (Connection conn = createOrdinaryConnection()) {
+ try (PreparedStatement s = conn.prepareStatement("SET STREAMING 1 FLUSH_FREQUENCY 10000")) {
+ s.addBatch();
+ }
+ }
+
+ return null;
+ }
+ }, SQLException.class, "Streaming control commands must be executed explicitly");
+ }
+
+ /**
+ * Check that there's nothing in cache.
+ */
+ private void assertCacheEmpty() {
+ assertEquals(0, cache().size(CachePeekMode.ALL));
+ }
+
+ /**
+ * @param conn Connection.
+ * @param sql Statement.
+ * @throws SQLException if failed.
+ */
+ private static void execute(Connection conn, String sql) throws SQLException {
+ try (Statement s = conn.createStatement()) {
+ s.execute(sql);
+ }
+ }
+
+ /**
+ * @return Active SQL client context.
+ */
+ private SqlClientContext sqlClientContext() {
+ assertNotNull(IndexingWithContext.cliCtx);
+
+ return IndexingWithContext.cliCtx;
+ }
+
+ /**
+ * Check that streaming state on target node is as expected.
+ * @param on Expected streaming state.
+ */
+ private void assertStreamingState(boolean on) {
+ SqlClientContext cliCtx = sqlClientContext();
+
+ assertEquals(on, cliCtx.isStream());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void assertStatementForbidden(String sql) {
+ batchSize = 1;
+
+ super.assertStatementForbidden(sql);
+ }
+
+ /**
+ *
+ */
+ private static final class IndexingWithContext extends IgniteH2Indexing {
+ /** Client context. */
+ static SqlClientContext cliCtx;
+
+ /** {@inheritDoc} */
+ @Override public List<Long> streamBatchedUpdateQuery(String schemaName, String qry, List<Object[]> params,
+ SqlClientContext cliCtx) throws IgniteCheckedException {
+ IndexingWithContext.cliCtx = cliCtx;
+
+ return super.streamBatchedUpdateQuery(schemaName, qry, params, cliCtx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<FieldsQueryCursor<List<?>>> querySqlFields(String schemaName, SqlFieldsQuery qry,
+ @Nullable SqlClientContext cliCtx, boolean keepBinary, boolean failOnMultipleStmts,
+ GridQueryCancel cancel) {
+ IndexingWithContext.cliCtx = cliCtx;
+
+ return super.querySqlFields(schemaName, qry, cliCtx, keepBinary, failOnMultipleStmts, cancel);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java
index ac9925d..779a022 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.jdbc.thin;
import java.sql.Array;
+import java.sql.BatchUpdateException;
import java.sql.Blob;
import java.sql.CallableStatement;
import java.sql.Clob;
@@ -33,17 +34,27 @@ import java.sql.SQLXML;
import java.sql.Savepoint;
import java.sql.Statement;
import java.sql.Struct;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Executor;
+import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
import org.apache.ignite.internal.processors.odbc.SqlStateCode;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBatchExecuteRequest;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBatchExecuteResult;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQuery;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryExecuteRequest;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcRequest;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcResponse;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcResult;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcStatementType;
import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.sql.command.SqlCommand;
+import org.apache.ignite.internal.sql.command.SqlSetStreamingCommand;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.lang.IgniteProductVersion;
@@ -79,6 +90,9 @@ public class JdbcThinConnection implements Connection {
/** Read-only flag. */
private boolean readOnly;
+ /** Streaming flag. */
+ private volatile boolean stream;
+
/** Current transaction holdability. */
private int holdability;
@@ -94,6 +108,15 @@ public class JdbcThinConnection implements Connection {
/** Connection properties. */
private ConnectionProperties connProps;
+ /** Batch size for streaming. */
+ private int streamBatchSize;
+
+ /** Batch for streaming. */
+ private List<JdbcQuery> streamBatch;
+
+ /** Last added query to recognize batches. */
+ private String lastStreamQry;
+
/**
* Creates new connection.
*
@@ -135,6 +158,86 @@ public class JdbcThinConnection implements Connection {
}
}
+ /**
+ * @return Whether this connection is streamed or not.
+ */
+ boolean isStream() {
+ return stream;
+ }
+
+ /**
+ * @param sql Statement.
+ * @param cmd Parsed form of {@code sql}.
+ * @throws SQLException if failed.
+ */
+ void executeNative(String sql, SqlCommand cmd) throws SQLException {
+ if (cmd instanceof SqlSetStreamingCommand) {
+ // If streaming is already on, we have to disable it first.
+ if (stream) {
+ // We have to send request regardless of actual batch size.
+ executeBatch(true);
+
+ stream = false;
+ }
+
+ boolean newVal = ((SqlSetStreamingCommand)cmd).isTurnOn();
+
+ // Actual ON, if needed.
+ if (newVal) {
+ sendRequest(new JdbcQueryExecuteRequest(JdbcStatementType.ANY_STATEMENT_TYPE,
+ schema, 1, 1, sql, null));
+
+ streamBatchSize = ((SqlSetStreamingCommand)cmd).batchSize();
+
+ stream = true;
+ }
+ }
+ else
+ throw IgniteQueryErrorCode.createJdbcSqlException("Unsupported native statement: " + sql,
+ IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+ }
+
+ /**
+ * Add another query for batched execution.
+ * @param sql Query.
+ * @param args Arguments.
+ */
+ void addBatch(String sql, List<Object> args) throws SQLException {
+ boolean newQry = (args == null || !F.eq(lastStreamQry, sql));
+
+ // Providing null as SQL here allows for recognizing subbatches on server and handling them more efficiently.
+ JdbcQuery q = new JdbcQuery(newQry ? sql : null, args != null ? args.toArray() : null);
+
+ if (streamBatch == null)
+ streamBatch = new ArrayList<>(streamBatchSize);
+
+ streamBatch.add(q);
+
+ // Null args means "addBatch(String)" was called on non-prepared Statement,
+ // we don't want to remember its query string.
+ lastStreamQry = (args != null ? sql : null);
+
+ if (streamBatch.size() == streamBatchSize)
+ executeBatch(false);
+ }
+
+ /**
+ * @param lastBatch Whether open data streamers must be flushed and closed after this batch.
+ * @throws SQLException if failed.
+ */
+ private void executeBatch(boolean lastBatch) throws SQLException {
+ JdbcBatchExecuteResult res = sendRequest(new JdbcBatchExecuteRequest(schema, streamBatch, lastBatch));
+
+ streamBatch = null;
+
+ lastStreamQry = null;
+
+ if (res.errorCode() != ClientListenerResponse.STATUS_SUCCESS) {
+ throw new BatchUpdateException(res.errorMessage(), IgniteQueryErrorCode.codeToSqlState(res.errorCode()),
+ res.errorCode(), res.updateCounts());
+ }
+ }
+
/** {@inheritDoc} */
@Override public Statement createStatement() throws SQLException {
return createStatement(TYPE_FORWARD_ONLY, CONCUR_READ_ONLY, HOLD_CURSORS_OVER_COMMIT);
@@ -277,6 +380,15 @@ public class JdbcThinConnection implements Connection {
if (isClosed())
return;
+ if (!F.isEmpty(streamBatch)) {
+ try {
+ executeBatch(true);
+ }
+ catch (SQLException e) {
+ LOG.log(Level.WARNING, "Exception during batch send on streamed connection close", e);
+ }
+ }
+
closed = true;
cliIo.close();
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java
index 23d3bbe..d18788c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java
@@ -39,8 +39,8 @@ import java.sql.Time;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Calendar;
-import org.apache.ignite.internal.processors.odbc.SqlStateCode;
import org.apache.ignite.internal.processors.odbc.SqlListenerUtils;
+import org.apache.ignite.internal.processors.odbc.SqlStateCode;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcMetaParamsRequest;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcMetaParamsResult;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQuery;
@@ -262,13 +262,23 @@ public class JdbcThinPreparedStatement extends JdbcThinStatement implements Prep
@Override public void addBatch() throws SQLException {
ensureNotClosed();
- if (batch == null) {
- batch = new ArrayList<>();
+ checkStatementEligibleForBatching(sql);
+
+ checkStatementBatchEmpty();
+
+ batchSize++;
+
+ if (conn.isStream())
+ conn.addBatch(sql, args);
+ else {
+ if (batch == null) {
+ batch = new ArrayList<>();
- batch.add(new JdbcQuery(sql, args.toArray(new Object[args.size()])));
+ batch.add(new JdbcQuery(sql, args.toArray(new Object[args.size()])));
+ }
+ else
+ batch.add(new JdbcQuery(null, args.toArray(new Object[args.size()])));
}
- else
- batch.add(new JdbcQuery(null, args.toArray(new Object[args.size()])));
args = null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java
index 9c41804..480edfa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java
@@ -38,14 +38,21 @@ import org.apache.ignite.internal.processors.odbc.SqlStateCode;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBatchExecuteRequest;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBatchExecuteResult;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBulkLoadAckResult;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBulkLoadBatchRequest;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQuery;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryExecuteMultipleStatementsResult;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryExecuteRequest;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryExecuteResult;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcResult;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcResultInfo;
-import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBulkLoadBatchRequest;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcStatementType;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
+import org.apache.ignite.internal.sql.SqlKeyword;
+import org.apache.ignite.internal.sql.SqlParseException;
+import org.apache.ignite.internal.sql.SqlParser;
+import org.apache.ignite.internal.sql.command.SqlCommand;
+import org.apache.ignite.internal.sql.command.SqlSetStreamingCommand;
+import org.apache.ignite.internal.util.typedef.F;
import static java.sql.ResultSet.CONCUR_READ_ONLY;
import static java.sql.ResultSet.FETCH_FORWARD;
@@ -79,6 +86,9 @@ public class JdbcThinStatement implements Statement {
/** Result set holdability*/
private final int resHoldability;
+ /** Batch size to keep track of number of items to return as fake update counters for executeBatch. */
+ protected int batchSize;
+
/** Batch. */
protected List<JdbcQuery> batch;
@@ -119,6 +129,40 @@ public class JdbcThinStatement implements Statement {
}
/**
+ * @param sql Query.
+ * @return Native {@link SqlCommand}, or {@code null} if parsing was not successful.
+ */
+ private SqlCommand tryParseNative(String sql) {
+ try {
+ return new SqlParser(schema, sql).nextCommand();
+ }
+ catch (SqlParseException e) {
+ return null;
+ }
+ }
+
+ /**
+ * @param sql Query.
+ * @return Whether it's worth trying to parse this statement on the client.
+ */
+ private static boolean isEligibleForNativeParsing(String sql) {
+ if (F.isEmpty(sql))
+ return false;
+
+ sql = sql.toUpperCase();
+
+ int setPos = sql.indexOf(SqlKeyword.SET);
+
+ if (setPos == -1)
+ return false;
+
+ int streamingPos = sql.indexOf(SqlKeyword.STREAMING);
+
+ // There must be at least one symbol between SET and STREAMING.
+ return streamingPos - setPos - SqlKeyword.SET.length() >= 1;
+ }
+
+ /**
* @param stmtType Expected statement type.
* @param sql Sql query.
* @param args Query parameters.
@@ -133,6 +177,36 @@ public class JdbcThinStatement implements Statement {
if (sql == null || sql.isEmpty())
throw new SQLException("SQL query is empty.");
+ checkStatementBatchEmpty();
+
+ SqlCommand nativeCmd = null;
+
+ if (stmtType != JdbcStatementType.SELECT_STATEMENT_TYPE && isEligibleForNativeParsing(sql))
+ nativeCmd = tryParseNative(sql);
+
+ if (nativeCmd != null) {
+ conn.executeNative(sql, nativeCmd);
+
+ resultSets = Collections.singletonList(resultSetForUpdate(0));
+
+ // If this command should be executed as native one, we do not treat it
+ // as an ordinary batch citizen.
+ return;
+ }
+
+ if (conn.isStream()) {
+ if (stmtType == JdbcStatementType.SELECT_STATEMENT_TYPE)
+ throw new SQLException("executeQuery() method is not allowed in streaming mode.",
+ SqlStateCode.INTERNAL_ERROR,
+ IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+
+ conn.addBatch(sql, args);
+
+ resultSets = Collections.singletonList(resultSetForUpdate(0));
+
+ return;
+ }
+
JdbcResult res0 = conn.sendRequest(new JdbcQueryExecuteRequest(stmtType, schema, pageSize,
maxRows, sql, args == null ? null : args.toArray(new Object[args.size()])));
@@ -158,11 +232,8 @@ public class JdbcThinStatement implements Statement {
boolean firstRes = true;
for(JdbcResultInfo rsInfo : resInfos) {
- if (!rsInfo.isQuery()) {
- resultSets.add(new JdbcThinResultSet(this, -1, pageSize,
- true, Collections.<List<Object>>emptyList(), false,
- conn.autoCloseServerCursor(), rsInfo.updateCount(), closeOnCompletion));
- }
+ if (!rsInfo.isQuery())
+ resultSets.add(resultSetForUpdate(rsInfo.updateCount()));
else {
if (firstRes) {
firstRes = false;
@@ -186,6 +257,26 @@ public class JdbcThinStatement implements Statement {
}
/**
+ * Check that user has not added anything to this statement's batch prior to turning streaming on.
+ * @throws SQLException if failed.
+ */
+ void checkStatementBatchEmpty() throws SQLException {
+ if (conn.isStream() && !F.isEmpty(batch))
+ throw new IgniteSQLException("Statement has non-empty batch (call executeBatch() or clearBatch() " +
+ "before enabling streaming).", IgniteQueryErrorCode.UNSUPPORTED_OPERATION).toJdbcException();
+ }
+
+ /**
+ * @param cnt Update counter.
+ * @return Result set for given update counter.
+ */
+ private JdbcThinResultSet resultSetForUpdate(long cnt) {
+ return new JdbcThinResultSet(this, -1, pageSize,
+ true, Collections.<List<Object>>emptyList(), false,
+ conn.autoCloseServerCursor(), cnt, closeOnCompletion);
+ }
+
+ /**
* Sends a file to server in batches via multiple {@link JdbcBulkLoadBatchRequest}s.
*
* @param cmdRes Result of invoking COPY command: contains server-parsed
@@ -469,16 +560,50 @@ public class JdbcThinStatement implements Statement {
@Override public void addBatch(String sql) throws SQLException {
ensureNotClosed();
+ checkStatementEligibleForBatching(sql);
+
+ checkStatementBatchEmpty();
+
+ batchSize++;
+
+ if (conn.isStream()) {
+ conn.addBatch(sql, null);
+
+ return;
+ }
+
if (batch == null)
batch = new ArrayList<>();
batch.add(new JdbcQuery(sql, null));
}
+ /**
+ * Check that we're not trying to add to connection's batch a native command (it should be executed explicitly).
+ * @param sql SQL command.
+ * @throws SQLException if there's an attempt to add a native command to JDBC batch.
+ */
+ void checkStatementEligibleForBatching(String sql) throws SQLException {
+ SqlCommand nativeCmd = null;
+
+ if (isEligibleForNativeParsing(sql))
+ nativeCmd = tryParseNative(sql);
+
+ if (nativeCmd != null) {
+ assert nativeCmd instanceof SqlSetStreamingCommand;
+
+ throw new SQLException("Streaming control commands must be executed explicitly - " +
+ "either via Statement.execute(String), or via using prepared statements.",
+ SqlStateCode.UNSUPPORTED_OPERATION);
+ }
+ }
+
/** {@inheritDoc} */
@Override public void clearBatch() throws SQLException {
ensureNotClosed();
+ batchSize = 0;
+
batch = null;
}
@@ -488,11 +613,21 @@ public class JdbcThinStatement implements Statement {
closeResults();
- if (batch == null || batch.isEmpty())
+ checkStatementBatchEmpty();
+
+ if (conn.isStream()) {
+ int[] res = new int[batchSize];
+
+ batchSize = 0;
+
+ return res;
+ }
+
+ if (F.isEmpty(batch))
throw new SQLException("Batch is empty.");
try {
- JdbcBatchExecuteResult res = conn.sendRequest(new JdbcBatchExecuteRequest(conn.getSchema(), batch));
+ JdbcBatchExecuteResult res = conn.sendRequest(new JdbcBatchExecuteRequest(conn.getSchema(), batch, false));
if (res.errorCode() != ClientListenerResponse.STATUS_SUCCESS) {
throw new BatchUpdateException(res.errorMessage(), IgniteQueryErrorCode.codeToSqlState(res.errorCode()),
@@ -502,6 +637,8 @@ public class JdbcThinStatement implements Statement {
return res.updateCounts();
}
finally {
+ batchSize = 0;
+
batch = null;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
index 7aa6c33..ba7258a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java
@@ -37,12 +37,14 @@ import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.processors.odbc.ClientListenerRequest;
import org.apache.ignite.internal.processors.odbc.SqlStateCode;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBatchExecuteRequest;
+import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQuery;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryCloseRequest;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryFetchRequest;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcQueryMetadataRequest;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcRequest;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcResponse;
import org.apache.ignite.internal.util.ipc.loopback.IpcClientTcpEndpoint;
+import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteProductVersion;
@@ -298,7 +300,8 @@ public class JdbcThinTcpIo {
ClientListenerProtocolVersion srvProtocolVer = ClientListenerProtocolVersion.create(maj, min, maintenance);
- if (VER_2_3_0.equals(srvProtocolVer) || VER_2_1_5.equals(srvProtocolVer))
+ if (VER_2_4_0.equals(srvProtocolVer) || VER_2_3_0.equals(srvProtocolVer) ||
+ VER_2_1_5.equals(srvProtocolVer))
handshake(srvProtocolVer);
else if (VER_2_1_0.equals(srvProtocolVer))
handshake_2_1_0();
@@ -391,9 +394,12 @@ public class JdbcThinTcpIo {
int cap;
if (req instanceof JdbcBatchExecuteRequest) {
- int cnt = Math.min(MAX_BATCH_QRY_CNT, ((JdbcBatchExecuteRequest)req).queries().size());
+ List<JdbcQuery> qrys = ((JdbcBatchExecuteRequest)req).queries();
- cap = cnt * DYNAMIC_SIZE_MSG_CAP;
+ int cnt = !F.isEmpty(qrys) ? Math.min(MAX_BATCH_QRY_CNT, qrys.size()) : 0;
+
+ // One additional byte for last batch flag.
+ cap = cnt * DYNAMIC_SIZE_MSG_CAP + 1;
}
else if (req instanceof JdbcQueryCloseRequest)
cap = QUERY_CLOSE_MSG_SIZE;
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
index b51e0b9..f61ccf9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
@@ -60,6 +60,7 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.odbc.SqlStateCode;
import org.apache.ignite.internal.processors.query.GridQueryIndexing;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.QueryUtils;
import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -612,10 +613,11 @@ public class JdbcConnection implements Connection {
PreparedStatement nativeStmt = prepareNativeStatement(sql);
- if (!idx.isInsertStatement(nativeStmt)) {
- throw new SQLException("Only INSERT operations are supported in streaming mode",
- SqlStateCode.INTERNAL_ERROR,
- IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+ try {
+ idx.checkStatementStreamable(nativeStmt);
+ }
+ catch (IgniteSQLException e) {
+ throw e.toJdbcException();
}
IgniteDataStreamer streamer = ignite().dataStreamer(cacheName);
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java
index 408f089..25f55f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java
@@ -55,7 +55,7 @@ class JdbcStreamedPreparedStatement extends JdbcPreparedStatement {
/** {@inheritDoc} */
@Override protected void execute0(String sql, Boolean isQuery) throws SQLException {
- assert isQuery != null && !isQuery;
+ assert isQuery == null || !isQuery;
long updCnt = conn.ignite().context().query().streamUpdateQuery(conn.cacheName(), conn.schemaName(),
streamer, sql, getArgs());
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index a834022..c5d68b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -629,7 +629,7 @@ public class IgniteCacheProxyImpl<K, V> extends AsyncSupportAdapter<IgniteCache<
boolean keepBinary = opCtxCall != null && opCtxCall.isKeepBinary();
- return ctx.kernalContext().query().querySqlFields(ctx, qry, keepBinary, false);
+ return ctx.kernalContext().query().querySqlFields(ctx, qry, null, keepBinary, false);
}
catch (Exception e) {
if (e instanceof CacheException)
@@ -662,7 +662,7 @@ public class IgniteCacheProxyImpl<K, V> extends AsyncSupportAdapter<IgniteCache<
if (qry instanceof SqlFieldsQuery)
return (FieldsQueryCursor<R>)ctx.kernalContext().query().querySqlFields(ctx, (SqlFieldsQuery)qry,
- keepBinary, true).get(0);
+ null, keepBinary, true).get(0);
if (qry instanceof ScanQuery)
return query((ScanQuery)qry, null, projection(qry.isLocal()));
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java
index 25e1049..73fd04f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcBatchExecuteRequest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.odbc.jdbc;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.ignite.binary.BinaryObjectException;
@@ -39,6 +40,12 @@ public class JdbcBatchExecuteRequest extends JdbcRequest {
private List<JdbcQuery> queries;
/**
+ * Last stream batch flag - whether open streamers on current connection
+ * must be flushed and closed after this batch.
+ */
+ private boolean lastStreamBatch;
+
+ /**
* Default constructor.
*/
public JdbcBatchExecuteRequest() {
@@ -49,13 +56,14 @@ public class JdbcBatchExecuteRequest extends JdbcRequest {
* @param schemaName Schema name.
* @param queries Queries.
*/
- public JdbcBatchExecuteRequest(String schemaName, List<JdbcQuery> queries) {
+ public JdbcBatchExecuteRequest(String schemaName, List<JdbcQuery> queries, boolean lastStreamBatch) {
super(BATCH_EXEC);
- assert !F.isEmpty(queries);
+ assert lastStreamBatch || !F.isEmpty(queries);
this.schemaName = schemaName;
this.queries = queries;
+ this.lastStreamBatch = lastStreamBatch;
}
/**
@@ -72,15 +80,29 @@ public class JdbcBatchExecuteRequest extends JdbcRequest {
return queries;
}
+ /**
+ * @return Last stream batch flag.
+ */
+ public boolean isLastStreamBatch() {
+ return lastStreamBatch;
+ }
+
/** {@inheritDoc} */
@Override public void writeBinary(BinaryWriterExImpl writer) throws BinaryObjectException {
super.writeBinary(writer);
writer.writeString(schemaName);
- writer.writeInt(queries.size());
- for (JdbcQuery q : queries)
- q.writeBinary(writer);
+ if (!F.isEmpty(queries)) {
+ writer.writeInt(queries.size());
+
+ for (JdbcQuery q : queries)
+ q.writeBinary(writer);
+ }
+ else
+ writer.writeInt(0);
+
+ writer.writeBoolean(lastStreamBatch);
}
/** {@inheritDoc} */
@@ -100,6 +122,14 @@ public class JdbcBatchExecuteRequest extends JdbcRequest {
queries.add(qry);
}
+
+ try {
+ if (reader.available() > 0)
+ lastStreamBatch = reader.readBoolean();
+ }
+ catch (IOException e) {
+ throw new BinaryObjectException(e);
+ }
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
index 5841a4d..d8f82f5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java
@@ -28,7 +28,7 @@ import org.apache.ignite.internal.processors.odbc.ClientListenerRequestHandler;
import org.apache.ignite.internal.util.GridSpinBusyLock;
/**
- * ODBC Connection Context.
+ * JDBC Connection Context.
*/
public class JdbcConnectionContext implements ClientListenerConnectionContext {
/** Version 2.1.0. */
@@ -38,13 +38,16 @@ public class JdbcConnectionContext implements ClientListenerConnectionContext {
private static final ClientListenerProtocolVersion VER_2_1_5 = ClientListenerProtocolVersion.create(2, 1, 5);
/** Version 2.3.1: added "multiple statements query" feature. */
- public static final ClientListenerProtocolVersion VER_2_3_0 = ClientListenerProtocolVersion.create(2, 3, 0);
+ static final ClientListenerProtocolVersion VER_2_3_0 = ClientListenerProtocolVersion.create(2, 3, 0);
/** Version 2.4.0: adds default values for columns feature. */
- public static final ClientListenerProtocolVersion VER_2_4_0 = ClientListenerProtocolVersion.create(2, 4, 0);
+ static final ClientListenerProtocolVersion VER_2_4_0 = ClientListenerProtocolVersion.create(2, 4, 0);
+
+ /** Version 2.5.0. */
+ private static final ClientListenerProtocolVersion VER_2_5_0 = ClientListenerProtocolVersion.create(2, 5, 0);
/** Current version. */
- private static final ClientListenerProtocolVersion CURRENT_VER = VER_2_4_0;
+ private static final ClientListenerProtocolVersion CURRENT_VER = VER_2_5_0;
/** Supported versions. */
private static final Set<ClientListenerProtocolVersion> SUPPORTED_VERS = new HashSet<>();
@@ -66,6 +69,8 @@ public class JdbcConnectionContext implements ClientListenerConnectionContext {
static {
SUPPORTED_VERS.add(CURRENT_VER);
+ SUPPORTED_VERS.add(VER_2_5_0);
+ SUPPORTED_VERS.add(VER_2_4_0);
SUPPORTED_VERS.add(VER_2_3_0);
SUPPORTED_VERS.add(VER_2_1_5);
SUPPORTED_VERS.add(VER_2_1_0);
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
index 59fc06b..baf4ec5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
@@ -53,6 +53,7 @@ import org.apache.ignite.internal.processors.query.GridQueryProperty;
import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.processors.query.SqlClientContext;
import org.apache.ignite.internal.util.GridSpinBusyLock;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.X;
@@ -88,6 +89,9 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
/** Kernel context. */
private final GridKernalContext ctx;
+ /** Client context. */
+ private final SqlClientContext cliCtx;
+
/** Logger. */
private final IgniteLogger log;
@@ -103,24 +107,6 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
/** Current bulk load processors. */
private final ConcurrentHashMap<Long, JdbcBulkLoadProcessor> bulkLoadRequests = new ConcurrentHashMap<>();
- /** Distributed joins flag. */
- private final boolean distributedJoins;
-
- /** Enforce join order flag. */
- private final boolean enforceJoinOrder;
-
- /** Collocated flag. */
- private final boolean collocated;
-
- /** Replicated only flag. */
- private final boolean replicatedOnly;
-
- /** Lazy query execution flag. */
- private final boolean lazy;
-
- /** Skip reducer on update flag. */
- private final boolean skipReducerOnUpdate;
-
/** Automatic close of cursors. */
private final boolean autoCloseCursors;
@@ -129,7 +115,6 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
/**
* Constructor.
- *
* @param ctx Context.
* @param busyLock Shutdown latch.
* @param maxCursors Maximum allowed cursors.
@@ -147,15 +132,20 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
boolean autoCloseCursors, boolean lazy, boolean skipReducerOnUpdate,
ClientListenerProtocolVersion protocolVer) {
this.ctx = ctx;
+
+ this.cliCtx = new SqlClientContext(
+ ctx,
+ distributedJoins,
+ enforceJoinOrder,
+ collocated,
+ replicatedOnly,
+ lazy,
+ skipReducerOnUpdate
+ );
+
this.busyLock = busyLock;
this.maxCursors = maxCursors;
- this.distributedJoins = distributedJoins;
- this.enforceJoinOrder = enforceJoinOrder;
- this.collocated = collocated;
- this.replicatedOnly = replicatedOnly;
this.autoCloseCursors = autoCloseCursors;
- this.lazy = lazy;
- this.skipReducerOnUpdate = skipReducerOnUpdate;
this.protocolVer = protocolVer;
log = ctx.log(getClass());
@@ -301,6 +291,8 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
}
bulkLoadRequests.clear();
+
+ U.close(cliCtx, log);
}
finally {
busyLock.leaveBusy();
@@ -326,6 +318,8 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
long qryId = QRY_ID_GEN.getAndIncrement();
+ assert !cliCtx.isStream();
+
try {
String sql = req.sqlQuery();
@@ -347,17 +341,17 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
qry = new SqlFieldsQueryEx(sql, false);
- if (skipReducerOnUpdate)
+ if (cliCtx.isSkipReducerOnUpdate())
((SqlFieldsQueryEx)qry).setSkipReducerOnUpdate(true);
}
qry.setArgs(req.arguments());
- qry.setDistributedJoins(distributedJoins);
- qry.setEnforceJoinOrder(enforceJoinOrder);
- qry.setCollocated(collocated);
- qry.setReplicatedOnly(replicatedOnly);
- qry.setLazy(lazy);
+ qry.setDistributedJoins(cliCtx.isDistributedJoins());
+ qry.setEnforceJoinOrder(cliCtx.isEnforceJoinOrder());
+ qry.setCollocated(cliCtx.isCollocated());
+ qry.setReplicatedOnly(cliCtx.isReplicatedOnly());
+ qry.setLazy(cliCtx.isLazy());
if (req.pageSize() <= 0)
return new JdbcResponse(IgniteQueryErrorCode.UNKNOWN, "Invalid fetch size: " + req.pageSize());
@@ -371,7 +365,7 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
qry.setSchema(schemaName);
- List<FieldsQueryCursor<List<?>>> results = ctx.query().querySqlFields(qry, true,
+ List<FieldsQueryCursor<List<?>>> results = ctx.query().querySqlFields(null, qry, cliCtx, true,
protocolVer.compareTo(VER_2_3_0) < 0);
FieldsQueryCursor<List<?>> fieldsCur = results.get(0);
@@ -569,11 +563,11 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
qry = new SqlFieldsQueryEx(q.sql(), false);
- qry.setDistributedJoins(distributedJoins);
- qry.setEnforceJoinOrder(enforceJoinOrder);
- qry.setCollocated(collocated);
- qry.setReplicatedOnly(replicatedOnly);
- qry.setLazy(lazy);
+ qry.setDistributedJoins(cliCtx.isDistributedJoins());
+ qry.setEnforceJoinOrder(cliCtx.isEnforceJoinOrder());
+ qry.setCollocated(cliCtx.isCollocated());
+ qry.setReplicatedOnly(cliCtx.isReplicatedOnly());
+ qry.setLazy(cliCtx.isLazy());
qry.setSchema(schemaName);
}
@@ -586,6 +580,9 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
if (qry != null)
executeBatchedQuery(qry, updCntsAcc, firstErr);
+ if (req.isLastStreamBatch())
+ cliCtx.disableStreaming();
+
int updCnts[] = U.toIntArray(updCntsAcc);
if (firstErr.isEmpty())
@@ -601,10 +598,21 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler {
* @param updCntsAcc Per query rows updates counter.
* @param firstErr First error data - code and message.
*/
+ @SuppressWarnings("ForLoopReplaceableByForEach")
private void executeBatchedQuery(SqlFieldsQueryEx qry, List<Integer> updCntsAcc,
IgniteBiTuple<Integer, String> firstErr) {
try {
- List<FieldsQueryCursor<List<?>>> qryRes = ctx.query().querySqlFields(qry, true, true);
+ if (cliCtx.isStream()) {
+ List<Long> cnt = ctx.query().streamBatchedUpdateQuery(qry.getSchema(), cliCtx, qry.getSql(),
+ qry.batchedArguments());
+
+ for (int i = 0; i < cnt.size(); i++)
+ updCntsAcc.add(cnt.get(i).intValue());
+
+ return;
+ }
+
+ List<FieldsQueryCursor<List<?>>> qryRes = ctx.query().querySqlFields(null, qry, cliCtx, true, true);
for (FieldsQueryCursor<List<?>> cur : qryRes) {
if (cur instanceof BulkLoadContextCursor)
http://git-wip-us.apache.org/repos/asf/ignite/blob/7366809e/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index 6b425a1..dedd075 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@ -76,33 +76,46 @@ public interface GridQueryIndexing {
* Detect whether SQL query should be executed in distributed or local manner and execute it.
* @param schemaName Schema name.
* @param qry Query.
+ * @param cliCtx Client context.
* @param keepBinary Keep binary flag.
* @param failOnMultipleStmts Whether an exception should be thrown for multiple statements query.
- * @param cancel Query cancel state handler.
- * @return Cursor.
+ * @param cancel Query cancel state handler. @return Cursor.
*/
- public List<FieldsQueryCursor<List<?>>> querySqlFields(String schemaName, SqlFieldsQuery qry, boolean keepBinary,
- boolean failOnMultipleStmts, GridQueryCancel cancel);
+ public List<FieldsQueryCursor<List<?>>> querySqlFields(String schemaName, SqlFieldsQuery qry,
+ SqlClientContext cliCtx, boolean keepBinary, boolean failOnMultipleStmts, GridQueryCancel cancel);
/**
- * Perform a MERGE statement using data streamer as receiver.
+ * Execute an INSERT statement using data streamer as receiver.
*
* @param schemaName Schema name.
* @param qry Query.
* @param params Query parameters.
* @param streamer Data streamer to feed data to.
- * @return Query result.
+ * @return Update counter.
* @throws IgniteCheckedException If failed.
*/
public long streamUpdateQuery(String schemaName, String qry, @Nullable Object[] params,
IgniteDataStreamer<?, ?> streamer) throws IgniteCheckedException;
/**
+ * Execute a batched INSERT statement using data streamer as receiver.
+ *
+ * @param schemaName Schema name.
+ * @param qry Query.
+ * @param params Query parameters.
+ * @param cliCtx Client connection context.
+ * @return Update counters.
+ * @throws IgniteCheckedException If failed.
+ */
+ public List<Long> streamBatchedUpdateQuery(String schemaName, String qry, List<Object[]> params,
+ SqlClientContext cliCtx) throws IgniteCheckedException;
+
+ /**
* Executes regular query.
*
* @param schemaName Schema name.
* @param cacheName Cache name.
- *@param qry Query.
+ * @param qry Query.
* @param filter Cache name and key filter.
* @param keepBinary Keep binary flag. @return Cursor.
*/
@@ -313,12 +326,11 @@ public interface GridQueryIndexing {
public String schema(String cacheName);
/**
- * Check if passed statement is insert statemtn.
+ * Check if passed statement is insert statement eligible for streaming, throw an {@link IgniteSQLException} if not.
*
* @param nativeStmt Native statement.
- * @return {@code True} if insert.
*/
- public boolean isInsertStatement(PreparedStatement nativeStmt);
+ public void checkStatementStreamable(PreparedStatement nativeStmt);
/**
* Return row cache cleaner.
[43/50] [abbrv] ignite git commit: IGNITE:7862: flatten plugin
updated to version 1.0.1
Posted by sb...@apache.org.
IGNITE:7862: flatten plugin updated to version 1.0.1
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1981dc66
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1981dc66
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1981dc66
Branch: refs/heads/ignite-zk
Commit: 1981dc66d9f3c6622c79d05b05cf97e203ccb416
Parents: 78e8627
Author: Peter Ivanov <mr...@gmail.com>
Authored: Fri Mar 16 15:07:04 2018 +0300
Committer: Nikolay Izhikov <ni...@apache.org>
Committed: Fri Mar 16 15:09:06 2018 +0300
----------------------------------------------------------------------
modules/platforms/dotnet/build.ps1 | 6 +++++-
parent/pom.xml | 2 +-
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1981dc66/modules/platforms/dotnet/build.ps1
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/build.ps1 b/modules/platforms/dotnet/build.ps1
index 70ce25d..59918b5 100644
--- a/modules/platforms/dotnet/build.ps1
+++ b/modules/platforms/dotnet/build.ps1
@@ -121,6 +121,10 @@ if (!$skipJava) {
echo "Maven detected at $mv."
}
+ # Install Maven Wrapper
+ cmd /c "$mv -N io.takari:maven:wrapper -Dmaven=3.5.2"
+ $mv = "mvnw.cmd"
+
# Run Maven
echo "Starting Java (Maven) build..."
@@ -256,4 +260,4 @@ if (!$skipNuGet) {
}
echo "NuGet packages created in '$pwd\$nupkgDir'."
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1981dc66/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index 5d85f0d..d60f6c2 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -612,7 +612,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
- <version>1.0.0-beta-3</version>
+ <version>1.0.1</version>
<configuration>
<flattenMode>minimum</flattenMode>
[03/50] [abbrv] ignite git commit: IGNITE-7566 .NET: Build scripts:
stop build when one of the steps fails
Posted by sb...@apache.org.
IGNITE-7566 .NET: Build scripts: stop build when one of the steps fails
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0936f162
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0936f162
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0936f162
Branch: refs/heads/ignite-zk
Commit: 0936f162b84d0ff701ac020405e7e44f55fae52e
Parents: 8c3731e
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Wed Mar 7 11:10:08 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Wed Mar 7 11:10:08 2018 +0300
----------------------------------------------------------------------
modules/platforms/dotnet/build-mono.sh | 5 +++++
modules/platforms/dotnet/build.sh | 5 +++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0936f162/modules/platforms/dotnet/build-mono.sh
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/build-mono.sh b/modules/platforms/dotnet/build-mono.sh
old mode 100644
new mode 100755
index 6f73eb7..a0a01c1
--- a/modules/platforms/dotnet/build-mono.sh
+++ b/modules/platforms/dotnet/build-mono.sh
@@ -20,10 +20,15 @@
# Builds Java and .NET with Mono
#
+# Fail on error.
+set -e
+
+# Build Java.
pushd .
cd ../../..
mvn clean package -DskipTests -Dmaven.javadoc.skip=true -Plgpl,-examples,-clean-libs,-release,-scala,-clientDocs
popd
+# Build .NET.
nuget restore Apache.Ignite.sln
msbuild Apache.Ignite.sln /p:RunCodeAnalysis=false
http://git-wip-us.apache.org/repos/asf/ignite/blob/0936f162/modules/platforms/dotnet/build.sh
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/build.sh b/modules/platforms/dotnet/build.sh
index fc5b344..1a58ff0 100755
--- a/modules/platforms/dotnet/build.sh
+++ b/modules/platforms/dotnet/build.sh
@@ -20,10 +20,15 @@
# Builds Java and .NET with .NET Core
#
+# Fail on error.
+#set -e
+
+# Build Java.
pushd .
cd ../../..
mvn clean package -DskipTests -Dmaven.javadoc.skip=true -Plgpl,-examples,-clean-libs,-release,-scala,-clientDocs
popd
+# Build .NET
cd Apache.Ignite.Core.Tests.DotNetCore
dotnet build
[22/50] [abbrv] ignite git commit: IGNITE-7751 Use throttling to
protect from checkpoint buffer overflow - Fixes #3611.
Posted by sb...@apache.org.
IGNITE-7751 Use throttling to protect from checkpoint buffer overflow - Fixes #3611.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/05d58bb6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/05d58bb6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/05d58bb6
Branch: refs/heads/ignite-zk
Commit: 05d58bb6efe35bfdd8bdb3e74dfd2627f6df7f9c
Parents: 8da7c9e
Author: Ivan Rakov <iv...@gmail.com>
Authored: Tue Mar 13 19:49:12 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Mar 13 19:49:12 2018 +0300
----------------------------------------------------------------------
.../internal/pagemem/PageIdAllocator.java | 4 +-
.../pagemem/impl/PageMemoryNoStoreImpl.java | 2 +-
.../GridCacheDatabaseSharedManager.java | 39 ++--
.../persistence/pagemem/PageMemoryImpl.java | 169 +++++++--------
.../pagemem/PagesWriteSpeedBasedThrottle.java | 16 +-
.../persistence/pagemem/PagesWriteThrottle.java | 42 ++--
.../pagemem/BPlusTreePageMemoryImplTest.java | 2 +-
.../BPlusTreeReuseListPageMemoryImplTest.java | 3 +-
...gnitePageMemReplaceDelayedWriteUnitTest.java | 5 +-
.../pagemem/IgniteThrottlingUnitTest.java | 1 -
.../pagemem/IndexStoragePageMemoryImplTest.java | 3 +-
.../pagemem/PageMemoryImplNoLoadTest.java | 3 +-
.../persistence/pagemem/PageMemoryImplTest.java | 211 ++++++++++++++++++-
13 files changed, 363 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
index ae7da14..c6aeabe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
@@ -38,11 +38,11 @@ public interface PageIdAllocator {
/**
* Allocates a page from the space for the given partition ID and the given flags.
*
- * @param cacheId Cache Group ID.
+ * @param grpId Cache Group ID.
* @param partId Partition ID.
* @return Allocated page ID.
*/
- public long allocatePage(int cacheId, int partId, byte flags) throws IgniteCheckedException;
+ public long allocatePage(int grpId, int partId, byte flags) throws IgniteCheckedException;
/**
* The given page is free now.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
index af1555e..7424af6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
@@ -254,7 +254,7 @@ public class PageMemoryNoStoreImpl implements PageMemory {
}
/** {@inheritDoc} */
- @Override public long allocatePage(int cacheId, int partId, byte flags) {
+ @Override public long allocatePage(int grpId, int partId, byte flags) {
memMetrics.incrementTotalAllocatedPages();
long relPtr = borrowFreePage();
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index d6a8a30..997f89a 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -193,6 +193,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
private final int walRebalanceThreshold = IgniteSystemProperties.getInteger(
IGNITE_PDS_WAL_REBALANCE_THRESHOLD, 500_000);
+ /** Value of property for throttling policy override. */
+ private final String throttlingPolicyOverride = IgniteSystemProperties.getString(
+ IgniteSystemProperties.IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED);
+
/** Checkpoint lock hold count. */
private static final ThreadLocal<Integer> CHECKPOINT_LOCK_HOLD_COUNT = new ThreadLocal<Integer>() {
@Override protected Integer initialValue() {
@@ -932,19 +936,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
chpBufSize = cacheSize;
}
- PageMemoryImpl.ThrottlingPolicy plc = persistenceCfg.isWriteThrottlingEnabled()
- ? PageMemoryImpl.ThrottlingPolicy.SPEED_BASED
- : PageMemoryImpl.ThrottlingPolicy.NONE;
-
- String val = IgniteSystemProperties.getString(IgniteSystemProperties.IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED);
-
- if (val != null) {
- if ("ratio".equalsIgnoreCase(val))
- plc = PageMemoryImpl.ThrottlingPolicy.TARGET_RATIO_BASED;
- else if ("speed".equalsIgnoreCase(val) || Boolean.valueOf(val))
- plc = PageMemoryImpl.ThrottlingPolicy.SPEED_BASED;
- }
-
GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker;
if (trackable)
@@ -985,7 +976,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
changeTracker,
this,
memMetrics,
- plc,
+ resolveThrottlingPolicy(),
this
);
@@ -994,6 +985,26 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
return pageMem;
}
+ /**
+ * Resolves throttling policy according to the settings.
+ */
+ @NotNull private PageMemoryImpl.ThrottlingPolicy resolveThrottlingPolicy() {
+ PageMemoryImpl.ThrottlingPolicy plc = persistenceCfg.isWriteThrottlingEnabled()
+ ? PageMemoryImpl.ThrottlingPolicy.SPEED_BASED
+ : PageMemoryImpl.ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY;
+
+ if (throttlingPolicyOverride != null) {
+ try {
+ plc = PageMemoryImpl.ThrottlingPolicy.valueOf(throttlingPolicyOverride.toUpperCase());
+ }
+ catch (IllegalArgumentException e) {
+ log.error("Incorrect value of IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED property: " +
+ throttlingPolicyOverride + ". Default throttling policy " + plc + " will be used.");
+ }
+ }
+ return plc;
+ }
+
/** {@inheritDoc} */
@Override protected void checkRegionEvictionProperties(DataRegionConfiguration regCfg, DataStorageConfiguration dbCfg)
throws IgniteCheckedException {
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index fa10a1f..9f979f5 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -80,6 +80,7 @@ import org.apache.ignite.internal.util.lang.GridInClosure3X;
import org.apache.ignite.internal.util.offheap.GridOffHeapOutOfMemoryException;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
+import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import static java.lang.Boolean.FALSE;
@@ -147,7 +148,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/** Page ID offset */
public static final int PAGE_ID_OFFSET = 16;
- /** Page cache ID offset. */
+ /** Page cache group ID offset. */
public static final int PAGE_CACHE_ID_OFFSET = 24;
/** Page pin counter offset. */
@@ -163,7 +164,7 @@ public class PageMemoryImpl implements PageMemoryEx {
* 8b Marker/timestamp
* 8b Relative pointer
* 8b Page ID
- * 4b Cache ID
+ * 4b Cache group ID
* 4b Pin count
* 8b Lock
* 8b Temporary buffer
@@ -176,6 +177,10 @@ public class PageMemoryImpl implements PageMemoryEx {
/** Tracking io. */
private static final TrackingPageIO trackingIO = TrackingPageIO.VERSIONS.latest();
+ /** Checkpoint pool overflow error message. */
+ public static final String CHECKPOINT_POOL_OVERFLOW_ERROR_MSG = "Failed to allocate temporary buffer for checkpoint " +
+ "(increase checkpointPageBufferSize configuration property)";
+
/** Page size. */
private final int sysPageSize;
@@ -277,7 +282,7 @@ public class PageMemoryImpl implements PageMemoryEx {
CheckpointLockStateChecker stateChecker,
DataRegionMetricsImpl memMetrics,
@Nullable ThrottlingPolicy throttlingPlc,
- @Nullable CheckpointWriteProgressSupplier cpProgressProvider
+ @NotNull CheckpointWriteProgressSupplier cpProgressProvider
) {
assert ctx != null;
assert pageSize > 0;
@@ -294,7 +299,7 @@ public class PageMemoryImpl implements PageMemoryEx {
null;
this.changeTracker = changeTracker;
this.stateChecker = stateChecker;
- this.throttlingPlc = throttlingPlc != null ? throttlingPlc : ThrottlingPolicy.NONE;
+ this.throttlingPlc = throttlingPlc != null ? throttlingPlc : ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY;
this.cpProgressProvider = cpProgressProvider;
storeMgr = ctx.pageStore();
@@ -363,22 +368,15 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/**
- *
+ * Resolves instance of {@link PagesWriteThrottlePolicy} according to chosen throttle policy.
*/
private void initWriteThrottle() {
- if (!isThrottlingEnabled())
- return;
-
- if (cpProgressProvider == null) {
- log.error("Write throttle can't start. CP progress provider not presented");
-
- throttlingPlc = ThrottlingPolicy.NONE;
- }
-
if (throttlingPlc == ThrottlingPolicy.SPEED_BASED)
writeThrottle = new PagesWriteSpeedBasedThrottle(this, cpProgressProvider, stateChecker, log);
- else if(throttlingPlc == ThrottlingPolicy.TARGET_RATIO_BASED)
- writeThrottle = new PagesWriteThrottle(this, cpProgressProvider, stateChecker);
+ else if (throttlingPlc == ThrottlingPolicy.TARGET_RATIO_BASED)
+ writeThrottle = new PagesWriteThrottle(this, cpProgressProvider, stateChecker, false);
+ else if (throttlingPlc == ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY)
+ writeThrottle = new PagesWriteThrottle(this, null, stateChecker, true);
}
/** {@inheritDoc} */
@@ -398,8 +396,8 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/** {@inheritDoc} */
- @Override public void releasePage(int cacheId, long pageId, long page) {
- Segment seg = segment(cacheId, pageId);
+ @Override public void releasePage(int grpId, long pageId, long page) {
+ Segment seg = segment(grpId, pageId);
seg.readLock().lock();
@@ -412,18 +410,18 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/** {@inheritDoc} */
- @Override public long readLock(int cacheId, long pageId, long page) {
- return readLockPage(page, new FullPageId(pageId, cacheId), false);
+ @Override public long readLock(int grpId, long pageId, long page) {
+ return readLockPage(page, new FullPageId(pageId, grpId), false);
}
/** {@inheritDoc} */
- @Override public void readUnlock(int cacheId, long pageId, long page) {
+ @Override public void readUnlock(int grpId, long pageId, long page) {
readUnlockPage(page);
}
/** {@inheritDoc} */
- @Override public long writeLock(int cacheId, long pageId, long page) {
- return writeLock(cacheId, pageId, page, false);
+ @Override public long writeLock(int grpId, long pageId, long page) {
+ return writeLock(grpId, pageId, page, false);
}
/** {@inheritDoc} */
@@ -432,14 +430,14 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/** {@inheritDoc} */
- @Override public long tryWriteLock(int cacheId, long pageId, long page) {
- return tryWriteLockPage(page, new FullPageId(pageId, cacheId), true);
+ @Override public long tryWriteLock(int grpId, long pageId, long page) {
+ return tryWriteLockPage(page, new FullPageId(pageId, grpId), true);
}
/** {@inheritDoc} */
- @Override public void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc,
+ @Override public void writeUnlock(int grpId, long pageId, long page, Boolean walPlc,
boolean dirtyFlag) {
- writeUnlock(cacheId, pageId, page, walPlc, dirtyFlag, false);
+ writeUnlock(grpId, pageId, page, walPlc, dirtyFlag, false);
}
/** {@inheritDoc} */
@@ -449,12 +447,12 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/** {@inheritDoc} */
- @Override public boolean isDirty(int cacheId, long pageId, long page) {
+ @Override public boolean isDirty(int grpId, long pageId, long page) {
return isDirty(page);
}
/** {@inheritDoc} */
- @Override public long allocatePage(int cacheId, int partId, byte flags) throws IgniteCheckedException {
+ @Override public long allocatePage(int grpId, int partId, byte flags) throws IgniteCheckedException {
assert flags == PageIdAllocator.FLAG_DATA && partId <= PageIdAllocator.MAX_PARTITION_ID ||
flags == PageIdAllocator.FLAG_IDX && partId == PageIdAllocator.INDEX_PARTITION :
"flags = " + flags + ", partId = " + partId;
@@ -464,19 +462,19 @@ public class PageMemoryImpl implements PageMemoryEx {
if (isThrottlingEnabled())
writeThrottle.onMarkDirty(false);
- long pageId = storeMgr.allocatePage(cacheId, partId, flags);
+ long pageId = storeMgr.allocatePage(grpId, partId, flags);
assert PageIdUtils.pageIndex(pageId) > 0; //it's crucial for tracking pages (zero page is super one)
// We need to allocate page in memory for marking it dirty to save it in the next checkpoint.
// Otherwise it is possible that on file will be empty page which will be saved at snapshot and read with error
// because there is no crc inside them.
- Segment seg = segment(cacheId, pageId);
+ Segment seg = segment(grpId, pageId);
DelayedDirtyPageWrite delayedWriter = delayedPageReplacementTracker != null
? delayedPageReplacementTracker.delayedPageWrite() : null;
- FullPageId fullId = new FullPageId(pageId, cacheId);
+ FullPageId fullId = new FullPageId(pageId, grpId);
seg.writeLock().lock();
@@ -484,15 +482,15 @@ public class PageMemoryImpl implements PageMemoryEx {
try {
long relPtr = seg.loadedPages.get(
- cacheId,
+ grpId,
PageIdUtils.effectivePageId(pageId),
- seg.partGeneration(cacheId, partId),
+ seg.partGeneration(grpId, partId),
INVALID_REL_PTR,
OUTDATED_REL_PTR
);
if (relPtr == OUTDATED_REL_PTR)
- relPtr = refreshOutdatedPage(seg, cacheId, pageId, false);
+ relPtr = refreshOutdatedPage(seg, grpId, pageId, false);
if (relPtr == INVALID_REL_PTR)
relPtr = seg.borrowOrAllocateFreePage(pageId);
@@ -528,7 +526,7 @@ public class PageMemoryImpl implements PageMemoryEx {
if (!ctx.wal().isAlwaysWriteFullPages())
ctx.wal().log(
new InitNewPageRecord(
- cacheId,
+ grpId,
pageId,
trackingIO.getType(),
trackingIO.getVersion(), pageId
@@ -539,7 +537,7 @@ public class PageMemoryImpl implements PageMemoryEx {
}
}
- seg.loadedPages.put(cacheId, PageIdUtils.effectivePageId(pageId), relPtr, seg.partGeneration(cacheId, partId));
+ seg.loadedPages.put(grpId, PageIdUtils.effectivePageId(pageId), relPtr, seg.partGeneration(grpId, partId));
}
catch (IgniteOutOfMemoryException oom) {
DataRegionConfiguration dataRegionCfg = getDataRegionConfiguration();
@@ -562,7 +560,7 @@ public class PageMemoryImpl implements PageMemoryEx {
}
//we have allocated 'tracking' page, we need to allocate regular one
- return isTrackingPage ? allocatePage(cacheId, partId, flags) : pageId;
+ return isTrackingPage ? allocatePage(grpId, partId, flags) : pageId;
}
/**
@@ -596,7 +594,7 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/** {@inheritDoc} */
- @Override public boolean freePage(int cacheId, long pageId) throws IgniteCheckedException {
+ @Override public boolean freePage(int grpId, long pageId) throws IgniteCheckedException {
assert false : "Free page should be never called directly when persistence is enabled.";
return false;
@@ -613,25 +611,25 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/** {@inheritDoc} */
- @Override public long acquirePage(int cacheId, long pageId) throws IgniteCheckedException {
- return acquirePage(cacheId, pageId, false);
+ @Override public long acquirePage(int grpId, long pageId) throws IgniteCheckedException {
+ return acquirePage(grpId, pageId, false);
}
/** {@inheritDoc} */
- @Override public long acquirePage(int cacheId, long pageId, boolean restore) throws IgniteCheckedException {
- FullPageId fullId = new FullPageId(pageId, cacheId);
+ @Override public long acquirePage(int grpId, long pageId, boolean restore) throws IgniteCheckedException {
+ FullPageId fullId = new FullPageId(pageId, grpId);
int partId = PageIdUtils.partId(pageId);
- Segment seg = segment(cacheId, pageId);
+ Segment seg = segment(grpId, pageId);
seg.readLock().lock();
try {
long relPtr = seg.loadedPages.get(
- cacheId,
+ grpId,
PageIdUtils.effectivePageId(pageId),
- seg.partGeneration(cacheId, partId),
+ seg.partGeneration(grpId, partId),
INVALID_REL_PTR,
INVALID_REL_PTR
);
@@ -660,9 +658,9 @@ public class PageMemoryImpl implements PageMemoryEx {
try {
// Double-check.
long relPtr = seg.loadedPages.get(
- cacheId,
+ grpId,
PageIdUtils.effectivePageId(pageId),
- seg.partGeneration(cacheId, partId),
+ seg.partGeneration(grpId, partId),
INVALID_REL_PTR,
OUTDATED_REL_PTR
);
@@ -688,10 +686,10 @@ public class PageMemoryImpl implements PageMemoryEx {
setDirty(fullId, absPtr, false, false);
seg.loadedPages.put(
- cacheId,
+ grpId,
PageIdUtils.effectivePageId(pageId),
relPtr,
- seg.partGeneration(cacheId, partId)
+ seg.partGeneration(grpId, partId)
);
long pageAddr = absPtr + PAGE_OVERHEAD;
@@ -722,7 +720,7 @@ public class PageMemoryImpl implements PageMemoryEx {
else if (relPtr == OUTDATED_REL_PTR) {
assert PageIdUtils.pageIndex(pageId) == 0 : fullId;
- relPtr = refreshOutdatedPage(seg, cacheId, pageId, false);
+ relPtr = refreshOutdatedPage(seg, grpId, pageId, false);
absPtr = seg.absolute(relPtr);
@@ -764,7 +762,7 @@ public class PageMemoryImpl implements PageMemoryEx {
ByteBuffer buf = wrapPointer(pageAddr, pageSize());
try {
- storeMgr.read(cacheId, pageId, buf);
+ storeMgr.read(grpId, pageId, buf);
}
catch (IgniteDataIntegrityViolationException ignore) {
U.warn(log, "Failed to read page (data integrity violation encountered, will try to " +
@@ -783,17 +781,17 @@ public class PageMemoryImpl implements PageMemoryEx {
/**
* @param seg Segment.
- * @param cacheId Cache ID.
+ * @param grpId Cache group ID.
* @param pageId Page ID.
* @param rmv {@code True} if page should be removed.
* @return Relative pointer to refreshed page.
*/
- private long refreshOutdatedPage(Segment seg, int cacheId, long pageId, boolean rmv) {
+ private long refreshOutdatedPage(Segment seg, int grpId, long pageId, boolean rmv) {
assert seg.writeLock().isHeldByCurrentThread();
- int tag = seg.partGeneration(cacheId, PageIdUtils.partId(pageId));
+ int tag = seg.partGeneration(grpId, PageIdUtils.partId(pageId));
- long relPtr = seg.loadedPages.refresh(cacheId, PageIdUtils.effectivePageId(pageId), tag);
+ long relPtr = seg.loadedPages.refresh(grpId, PageIdUtils.effectivePageId(pageId), tag);
long absPtr = seg.absolute(relPtr);
@@ -815,13 +813,13 @@ public class PageMemoryImpl implements PageMemoryEx {
}
if (rmv)
- seg.loadedPages.remove(cacheId, PageIdUtils.effectivePageId(pageId));
+ seg.loadedPages.remove(grpId, PageIdUtils.effectivePageId(pageId));
if (seg.segCheckpointPages != null)
- seg.segCheckpointPages.remove(new FullPageId(pageId, cacheId));
+ seg.segCheckpointPages.remove(new FullPageId(pageId, grpId));
if (seg.dirtyPages != null)
- seg.dirtyPages.remove(new FullPageId(pageId, cacheId));
+ seg.dirtyPages.remove(new FullPageId(pageId, grpId));
return relPtr;
}
@@ -991,7 +989,7 @@ public class PageMemoryImpl implements PageMemoryEx {
memMetrics.resetDirtyPages();
- if (isThrottlingEnabled())
+ if (throttlingPlc != ThrottlingPolicy.DISABLED)
writeThrottle.onBeginCheckpoint();
return new GridMultiCollectionWrapper<>(collections);
@@ -1001,7 +999,7 @@ public class PageMemoryImpl implements PageMemoryEx {
* @return {@code True} if throttling is enabled.
*/
private boolean isThrottlingEnabled() {
- return throttlingPlc != ThrottlingPolicy.NONE;
+ return throttlingPlc != ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY && throttlingPlc != ThrottlingPolicy.DISABLED;
}
/** {@inheritDoc} */
@@ -1013,7 +1011,7 @@ public class PageMemoryImpl implements PageMemoryEx {
for (Segment seg : segments)
seg.segCheckpointPages = null;
- if (isThrottlingEnabled())
+ if (throttlingPlc != ThrottlingPolicy.DISABLED)
writeThrottle.onFinishCheckpoint();
}
@@ -1340,8 +1338,8 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/** {@inheritDoc} */
- @Override public long readLockForce(int cacheId, long pageId, long page) {
- return readLockPage(page, new FullPageId(pageId, cacheId), true);
+ @Override public long readLockForce(int grpId, long pageId, long page) {
+ return readLockPage(page, new FullPageId(pageId, grpId), true);
}
/**
@@ -1400,9 +1398,7 @@ public class PageMemoryImpl implements PageMemoryEx {
if (tmpRelPtr == INVALID_REL_PTR) {
rwLock.writeUnlock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
- throw new IgniteException(
- "Failed to allocate temporary buffer for checkpoint " +
- "(increase checkpointPageBufferSize configuration property)");
+ throw new IgniteException(CHECKPOINT_POOL_OVERFLOW_ERROR_MSG + ": " + memMetrics.getName());
}
// Pin the page until checkpoint is not finished.
@@ -1472,7 +1468,7 @@ public class PageMemoryImpl implements PageMemoryEx {
try {
rwLock.writeUnlock(page + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
- if (isThrottlingEnabled() && !restore && markDirty && !wasDirty)
+ if (throttlingPlc != ThrottlingPolicy.DISABLED && !restore && markDirty && !wasDirty)
writeThrottle.onMarkDirty(isInCheckpoint(fullId));
}
catch (AssertionError ex) {
@@ -1847,7 +1843,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/** Initial partition generation. */
private static final int INIT_PART_GENERATION = 1;
- /** Maps partition (cacheId, partId) to its generation. Generation is 1-based incrementing partition counter. */
+ /** Maps partition (grpId, partId) to its generation. Generation is 1-based incrementing partition counter. */
private final Map<GroupPartitionId, Integer> partGenerationMap = new HashMap<>();
/** */
@@ -1882,7 +1878,7 @@ public class PageMemoryImpl implements PageMemoryEx {
pool = new PagePool(idx, poolRegion, null);
- maxDirtyPages = throttlingPlc != ThrottlingPolicy.NONE
+ maxDirtyPages = throttlingPlc != ThrottlingPolicy.DISABLED
? pool.pages() * 3 / 4
: Math.min(pool.pages() * 2 / 3, cpPoolPages);
}
@@ -2599,37 +2595,37 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/**
- * Reads cache ID from the page at the given absolute pointer.
+ * Reads cache group ID from the page at the given absolute pointer.
*
* @param absPtr Absolute memory pointer to the page header.
- * @return Cache ID written to the page.
+ * @return Cache group ID written to the page.
*/
- private static int readPageCacheId(final long absPtr) {
+ private static int readPageGroupId(final long absPtr) {
return GridUnsafe.getInt(absPtr + PAGE_CACHE_ID_OFFSET);
}
/**
- * Writes cache ID from the page at the given absolute pointer.
+ * Writes cache group ID from the page at the given absolute pointer.
*
* @param absPtr Absolute memory pointer to the page header.
- * @param cacheId Cache ID to write.
+ * @param grpId Cache group ID to write.
*/
- private static void pageCacheId(final long absPtr, final int cacheId) {
- GridUnsafe.putInt(absPtr + PAGE_CACHE_ID_OFFSET, cacheId);
+ private static void pageGroupId(final long absPtr, final int grpId) {
+ GridUnsafe.putInt(absPtr + PAGE_CACHE_ID_OFFSET, grpId);
}
/**
- * Reads page ID and cache ID from the page at the given absolute pointer.
+ * Reads page ID and cache group ID from the page at the given absolute pointer.
*
* @param absPtr Absolute memory pointer to the page header.
* @return Full page ID written to the page.
*/
private static FullPageId fullPageId(final long absPtr) {
- return new FullPageId(readPageId(absPtr), readPageCacheId(absPtr));
+ return new FullPageId(readPageId(absPtr), readPageGroupId(absPtr));
}
/**
- * Writes page ID and cache ID from the page at the given absolute pointer.
+ * Writes page ID and cache group ID from the page at the given absolute pointer.
*
* @param absPtr Absolute memory pointer to the page header.
* @param fullPageId Full page ID to write.
@@ -2637,7 +2633,7 @@ public class PageMemoryImpl implements PageMemoryEx {
private static void fullPageId(final long absPtr, final FullPageId fullPageId) {
pageId(absPtr, fullPageId.pageId());
- pageCacheId(absPtr, fullPageId.groupId());
+ pageGroupId(absPtr, fullPageId.groupId());
}
}
@@ -2734,8 +2730,13 @@ public class PageMemoryImpl implements PageMemoryEx {
* Throttling enabled and its type enum.
*/
public enum ThrottlingPolicy {
- /** Not throttled. */NONE,
- /** Target ratio based: CP progress is used as border. */ TARGET_RATIO_BASED,
- /** Speed based. CP writting speed and estimated ideal speed are used as border */ SPEED_BASED
+ /** All ways of throttling are disabled. */
+ DISABLED,
+ /** Only exponential throttling is used to protect from CP buffer overflow. */
+ CHECKPOINT_BUFFER_ONLY,
+ /** Target ratio based: CP progress is used as border. */
+ TARGET_RATIO_BASED,
+ /** Speed based. CP writting speed and estimated ideal speed are used as border */
+ SPEED_BASED
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java
index aaf5471..68fa529 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java
@@ -196,21 +196,23 @@ public class PagesWriteSpeedBasedThrottle implements PagesWriteThrottlePolicy {
markDirtySpeed,
curCpWriteSpeed);
- level = ThrottleMode.LIMITED;
+ level = throttleParkTimeNs == 0 ? ThrottleMode.NO : ThrottleMode.LIMITED;
}
}
}
- if (level == ThrottleMode.NO) {
- exponentialBackoffCntr.set(0);
-
- throttleParkTimeNs = 0;
- }
- else if (level == ThrottleMode.EXPONENTIAL) {
+ if (level == ThrottleMode.EXPONENTIAL) {
int exponent = exponentialBackoffCntr.getAndIncrement();
throttleParkTimeNs = (long)(STARTING_THROTTLE_NANOS * Math.pow(BACKOFF_RATIO, exponent));
}
+ else {
+ if (isPageInCheckpoint)
+ exponentialBackoffCntr.set(0);
+
+ if (level == ThrottleMode.NO)
+ throttleParkTimeNs = 0;
+ }
if (throttleParkTimeNs > 0) {
recurrentLogIfNeed();
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
index 78e5344..166cdcd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
@@ -32,6 +32,9 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
/** Database manager. */
private final CheckpointWriteProgressSupplier cpProgress;
+ /** If true, throttle will only protect from checkpoint buffer overflow, not from dirty pages ratio cap excess. */
+ private final boolean throttleOnlyPagesInCheckpoint;
+
/** Checkpoint lock state checker. */
private CheckpointLockStateChecker stateChecker;
@@ -41,30 +44,36 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
/** Backoff ratio. Each next park will be this times longer. */
private static final double BACKOFF_RATIO = 1.05;
- /** Exponential backoff counter. */
- private final AtomicInteger exponentialBackoffCntr = new AtomicInteger(0);
+ /** Counter for dirty pages ratio throttling. */
+ private final AtomicInteger notInCheckpointBackoffCntr = new AtomicInteger(0);
+
+ /** Counter for checkpoint buffer usage ratio throttling (we need a separate one due to IGNITE-7751). */
+ private final AtomicInteger inCheckpointBackoffCntr = new AtomicInteger(0);
+
/**
* @param pageMemory Page memory.
* @param cpProgress Database manager.
* @param stateChecker checkpoint lock state checker.
+ * @param throttleOnlyPagesInCheckpoint If true, throttle will only protect from checkpoint buffer overflow.
*/
public PagesWriteThrottle(PageMemoryImpl pageMemory,
CheckpointWriteProgressSupplier cpProgress,
- CheckpointLockStateChecker stateChecker) {
+ CheckpointLockStateChecker stateChecker,
+ boolean throttleOnlyPagesInCheckpoint
+ ) {
this.pageMemory = pageMemory;
this.cpProgress = cpProgress;
this.stateChecker = stateChecker;
+ this.throttleOnlyPagesInCheckpoint = throttleOnlyPagesInCheckpoint;
+
+ if (!throttleOnlyPagesInCheckpoint)
+ assert cpProgress != null : "cpProgress must be not null if ratio based throttling mode is used";
}
/** {@inheritDoc} */
@Override public void onMarkDirty(boolean isPageInCheckpoint) {
assert stateChecker.checkpointLockIsHeldByThread();
- AtomicInteger writtenPagesCntr = cpProgress.writtenPagesCounter();
-
- if (writtenPagesCntr == null)
- return; // Don't throttle if checkpoint is not running.
-
boolean shouldThrottle = false;
if (isPageInCheckpoint) {
@@ -73,7 +82,12 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
shouldThrottle = pageMemory.checkpointBufferPagesCount() > checkpointBufLimit;
}
- if (!shouldThrottle) {
+ if (!shouldThrottle && !throttleOnlyPagesInCheckpoint) {
+ AtomicInteger writtenPagesCntr = cpProgress.writtenPagesCounter();
+
+ if (writtenPagesCntr == null)
+ return; // Don't throttle if checkpoint is not running.
+
int cpWrittenPages = writtenPagesCntr.get();
int cpTotalPages = cpProgress.currentCheckpointPagesCount();
@@ -92,13 +106,15 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
}
}
+ AtomicInteger cntr = isPageInCheckpoint ? inCheckpointBackoffCntr : notInCheckpointBackoffCntr;
+
if (shouldThrottle) {
- int throttleLevel = exponentialBackoffCntr.getAndIncrement();
+ int throttleLevel = cntr.getAndIncrement();
LockSupport.parkNanos((long)(STARTING_THROTTLE_NANOS * Math.pow(BACKOFF_RATIO, throttleLevel)));
}
else
- exponentialBackoffCntr.set(0);
+ cntr.set(0);
}
/** {@inheritDoc} */
@@ -107,6 +123,8 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
/** {@inheritDoc} */
@Override public void onFinishCheckpoint() {
- exponentialBackoffCntr.set(0);
+ inCheckpointBackoffCntr.set(0);
+
+ notInCheckpointBackoffCntr.set(0);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
index a305b7f..3737204 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
@@ -76,7 +76,7 @@ public class BPlusTreePageMemoryImplTest extends BPlusTreeSelfTest {
},
() -> true,
new DataRegionMetricsImpl(new DataRegionConfiguration()),
- PageMemoryImpl.ThrottlingPolicy.NONE,
+ PageMemoryImpl.ThrottlingPolicy.DISABLED,
null
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
index 1fc34c5..0c786ad 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
@@ -23,7 +23,6 @@ import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.database.BPlusTreeReuseSelfTest;
@@ -77,7 +76,7 @@ public class BPlusTreeReuseListPageMemoryImplTest extends BPlusTreeReuseSelfTest
},
() -> true,
new DataRegionMetricsImpl(new DataRegionConfiguration()),
- PageMemoryImpl.ThrottlingPolicy.NONE,
+ PageMemoryImpl.ThrottlingPolicy.DISABLED,
null
);
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java
index a87a61e..c6f42e1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java
@@ -223,9 +223,8 @@ public class IgnitePageMemReplaceDelayedWriteUnitTest {
DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
- PageMemoryImpl memory
- = new PageMemoryImpl(provider, sizes, sctx, pageSize,
- pageWriter, null, () -> true, memMetrics, null, null);
+ PageMemoryImpl memory = new PageMemoryImpl(provider, sizes, sctx, pageSize,
+ pageWriter, null, () -> true, memMetrics, PageMemoryImpl.ThrottlingPolicy.DISABLED, null);
memory.start();
return memory;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java
index 1cef087..f9ca7e4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgniteThrottlingUnitTest.java
@@ -23,7 +23,6 @@ import java.util.concurrent.locks.LockSupport;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
import org.apache.ignite.internal.processors.cache.persistence.CheckpointWriteProgressSupplier;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.logger.NullLogger;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
index 4495dc1..9087b1c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
@@ -24,7 +24,6 @@ import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.database.IndexStorageSelfTest;
@@ -92,7 +91,7 @@ public class IndexStoragePageMemoryImplTest extends IndexStorageSelfTest {
},
() -> true,
new DataRegionMetricsImpl(new DataRegionConfiguration()),
- PageMemoryImpl.ThrottlingPolicy.NONE,
+ PageMemoryImpl.ThrottlingPolicy.DISABLED,
null
);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
index 3c169be..34fd93b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
@@ -86,7 +86,8 @@ public class PageMemoryImplNoLoadTest extends PageMemoryNoLoadSelfTest {
}
},
new DataRegionMetricsImpl(new DataRegionConfiguration()),
- PageMemoryImpl.ThrottlingPolicy.NONE, null
+ PageMemoryImpl.ThrottlingPolicy.DISABLED,
+ null
);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05d58bb6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
index 8f0ef39..31af118 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
@@ -17,24 +17,40 @@
package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageUtils;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointWriteProgressSupplier;
import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor;
import org.apache.ignite.internal.util.lang.GridInClosure3X;
import org.apache.ignite.plugin.PluginProvider;
+import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.GridTestKernalContext;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.testframework.junits.logger.GridTestLog4jLogger;
+import org.mockito.Mockito;
+
+import static org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.CHECKPOINT_POOL_OVERFLOW_ERROR_MSG;
/**
*
@@ -46,11 +62,14 @@ public class PageMemoryImplTest extends GridCommonAbstractTest {
/** Page size. */
private static final int PAGE_SIZE = 1024;
+ /** Max memory size. */
+ private static final int MAX_SIZE = 128;
+
/**
* @throws Exception if failed.
*/
public void testThatAllocationTooMuchPagesCauseToOOMException() throws Exception {
- PageMemoryImpl memory = createPageMemory();
+ PageMemoryImpl memory = createPageMemory(PageMemoryImpl.ThrottlingPolicy.DISABLED);
try {
while (!Thread.currentThread().isInterrupted())
@@ -64,15 +83,186 @@ public class PageMemoryImplTest extends GridCommonAbstractTest {
}
/**
- *
+ * @throws Exception If failed.
+ */
+ public void testCheckpointBufferOverusageDontCauseWriteLockLeak() throws Exception {
+ PageMemoryImpl memory = createPageMemory(PageMemoryImpl.ThrottlingPolicy.DISABLED);
+
+ List<FullPageId> pages = new ArrayList<>();
+
+ try {
+ while (!Thread.currentThread().isInterrupted()) {
+ long pageId = memory.allocatePage(1, PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX);
+
+ FullPageId fullPageId = new FullPageId(pageId, 1);
+
+ pages.add(fullPageId);
+
+ acquireAndReleaseWriteLock(memory, fullPageId); //to set page id, otherwise we would fail with assertion error
+ }
+ }
+ catch (IgniteOutOfMemoryException ignore) {
+ //Success
+ }
+
+ memory.beginCheckpoint();
+
+ final AtomicReference<FullPageId> lastPage = new AtomicReference<>();
+
+ try {
+ for (FullPageId fullPageId : pages) {
+ lastPage.set(fullPageId);
+
+ acquireAndReleaseWriteLock(memory, fullPageId);
+ }
+ }
+ catch (Exception ex) {
+ assertTrue(ex.getMessage().startsWith(CHECKPOINT_POOL_OVERFLOW_ERROR_MSG));
+ }
+
+ memory.finishCheckpoint();
+
+ GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ acquireAndReleaseWriteLock(memory, lastPage.get()); //we should be able get lock again
+ }
+ catch (IgniteCheckedException e) {
+ throw new AssertionError(e);
+ }
+ }
+ }).get(getTestTimeout());
+ }
+
+ /**
+ * Tests that checkpoint buffer won't be overflowed with enabled CHECKPOINT_BUFFER_ONLY throttling.
+ * @throws Exception If failed.
*/
- private PageMemoryImpl createPageMemory() throws Exception {
+ public void testCheckpointBufferCantOverflowMixedLoad() throws Exception {
+ testCheckpointBufferCantOverflowWithThrottlingMixedLoad(PageMemoryImpl.ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY);
+ }
+
+ /**
+ * Tests that checkpoint buffer won't be overflowed with enabled SPEED_BASED throttling.
+ * @throws Exception If failed.
+ */
+ public void testCheckpointBufferCantOverflowMixedLoadSpeedBased() throws Exception {
+ testCheckpointBufferCantOverflowWithThrottlingMixedLoad(PageMemoryImpl.ThrottlingPolicy.SPEED_BASED);
+ }
+
+ /**
+ * Tests that checkpoint buffer won't be overflowed with enabled TARGET_RATIO_BASED throttling.
+ * @throws Exception If failed.
+ */
+ public void testCheckpointBufferCantOverflowMixedLoadRatioBased() throws Exception {
+ testCheckpointBufferCantOverflowWithThrottlingMixedLoad(PageMemoryImpl.ThrottlingPolicy.TARGET_RATIO_BASED);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void testCheckpointBufferCantOverflowWithThrottlingMixedLoad(PageMemoryImpl.ThrottlingPolicy plc) throws Exception {
+ PageMemoryImpl memory = createPageMemory(plc);
+
+ List<FullPageId> pages = new ArrayList<>();
+
+ for (int i = 0; i < (MAX_SIZE - 10) * MB / PAGE_SIZE / 2; i++) {
+ long pageId = memory.allocatePage(1, PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX);
+
+ FullPageId fullPageId = new FullPageId(pageId, 1);
+
+ pages.add(fullPageId);
+
+ acquireAndReleaseWriteLock(memory, fullPageId);
+ }
+
+ memory.beginCheckpoint();
+
+ CheckpointMetricsTracker mockTracker = Mockito.mock(CheckpointMetricsTracker.class);
+
+ for (FullPageId checkpointPage : pages)
+ memory.getForCheckpoint(checkpointPage, ByteBuffer.allocate(PAGE_SIZE), mockTracker);
+
+ memory.finishCheckpoint();
+
+ for (int i = (int)((MAX_SIZE - 10) * MB / PAGE_SIZE / 2); i < (MAX_SIZE - 20) * MB / PAGE_SIZE; i++) {
+ long pageId = memory.allocatePage(1, PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX);
+
+ FullPageId fullPageId = new FullPageId(pageId, 1);
+
+ pages.add(fullPageId);
+
+ acquireAndReleaseWriteLock(memory, fullPageId);
+ }
+
+ memory.beginCheckpoint();
+
+ Collections.shuffle(pages); // Mix pages in checkpoint with clean pages
+
+ AtomicBoolean stop = new AtomicBoolean(false);
+
+ try {
+ GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ for (FullPageId page : pages) {
+ if (ThreadLocalRandom.current().nextDouble() < 0.5) // Mark dirty 50% of pages
+ try {
+ acquireAndReleaseWriteLock(memory, page);
+
+ if (stop.get())
+ break;
+ }
+ catch (IgniteCheckedException e) {
+ log.error("runAsync ended with exception", e);
+
+ fail();
+ }
+ }
+ }
+ }).get(5_000);
+ }
+ catch (IgniteFutureTimeoutCheckedException ex) {
+ // Expected.
+ }
+ finally {
+ stop.set(true);
+ }
+
+ memory.finishCheckpoint();
+ }
+
+ /**
+ * @param memory Memory.
+ * @param fullPageId Full page id.
+ * @throws IgniteCheckedException If acquiring lock failed.
+ */
+ private void acquireAndReleaseWriteLock(PageMemoryImpl memory, FullPageId fullPageId) throws IgniteCheckedException {
+ long page = memory.acquirePage(1, fullPageId.pageId());
+
+ long address = memory.writeLock(1, fullPageId.pageId(), page);
+
+ PageIO.setPageId(address, fullPageId.pageId());
+
+ PageIO.setType(address, PageIO.T_BPLUS_META);
+
+ PageUtils.putShort(address, PageIO.VER_OFF, (short)1);
+
+ memory.writeUnlock(1, fullPageId.pageId(), page, Boolean.FALSE, true);
+
+ memory.releasePage(1, fullPageId.pageId(), page);
+ }
+
+ /**
+ * @param throttlingPlc Throttling Policy.
+ * @throws Exception If creating mock failed.
+ */
+ private PageMemoryImpl createPageMemory(PageMemoryImpl.ThrottlingPolicy throttlingPlc) throws Exception {
long[] sizes = new long[5];
for (int i = 0; i < sizes.length; i++)
- sizes[i] = 1024 * MB / 4;
+ sizes[i] = MAX_SIZE * MB / 4;
- sizes[4] = 10 * MB;
+ sizes[4] = 5 * MB;
DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
@@ -101,6 +291,13 @@ public class PageMemoryImplTest extends GridCommonAbstractTest {
null
);
+ CheckpointWriteProgressSupplier noThrottle = Mockito.mock(CheckpointWriteProgressSupplier.class);
+
+ Mockito.when(noThrottle.currentCheckpointPagesCount()).thenReturn(1_000_000);
+ Mockito.when(noThrottle.evictedPagesCntr()).thenReturn(new AtomicInteger(0));
+ Mockito.when(noThrottle.syncedPagesCounter()).thenReturn(new AtomicInteger(1_000_000));
+ Mockito.when(noThrottle.writtenPagesCounter()).thenReturn(new AtomicInteger(1_000_000));
+
PageMemoryImpl mem = new PageMemoryImpl(
provider,
sizes,
@@ -118,8 +315,8 @@ public class PageMemoryImplTest extends GridCommonAbstractTest {
}
},
new DataRegionMetricsImpl(igniteCfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration()),
- PageMemoryImpl.ThrottlingPolicy.NONE,
- null
+ throttlingPlc,
+ noThrottle
);
mem.start();
[44/50] [abbrv] ignite git commit: .NET: Fix IgniteUuid type id
(broken by IGNITE-7756 changes)
Posted by sb...@apache.org.
.NET: Fix IgniteUuid type id (broken by IGNITE-7756 changes)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/094f93b4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/094f93b4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/094f93b4
Branch: refs/heads/ignite-zk
Commit: 094f93b4160b9a0d41ad958bc4ec5a737188e44e
Parents: 1981dc6
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Fri Mar 16 16:02:40 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Fri Mar 16 16:02:40 2018 +0300
----------------------------------------------------------------------
.../dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/094f93b4/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs
index 8067a39..fab236c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs
@@ -167,8 +167,8 @@ namespace Apache.Ignite.Core.Impl.Binary
/** Type: object written with Java OptimizedMarshaller. */
public const byte OptimizedMarshaller = 254;
- /** Type: platform object proxy. */
- public const int IgniteUuid = 2018070327;
+ /** Type: Ignite UUID. */
+ public const int IgniteUuid = 63;
/** Type ids. */
private static readonly Dictionary<Type, byte> TypeIds = new Dictionary<Type, byte>
[09/50] [abbrv] ignite git commit: IGNITE-7803 Fixed regression in
tests.
Posted by sb...@apache.org.
IGNITE-7803 Fixed regression in tests.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/90c0af88
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/90c0af88
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/90c0af88
Branch: refs/heads/ignite-zk
Commit: 90c0af887508ebace570cd13e7cf6b4369a995dc
Parents: 6dbcf7e
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Thu Mar 8 00:17:20 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Mar 8 00:17:20 2018 +0700
----------------------------------------------------------------------
.../client/impl/ClientCacheFlagsCodecTest.java | 35 +++----
.../internal/client/GridClientCacheFlag.java | 48 ++++++++++
.../impl/connection/GridClientConnection.java | 18 +---
.../connection/GridClientNioTcpConnection.java | 6 +-
.../handlers/cache/GridCacheCommandHandler.java | 96 +++++++++-----------
.../cache/GridCacheCommandHandlerSelfTest.java | 9 +-
.../http/jetty/GridJettyRestHandler.java | 4 +-
7 files changed, 120 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/clients/src/test/java/org/apache/ignite/internal/client/impl/ClientCacheFlagsCodecTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/impl/ClientCacheFlagsCodecTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/impl/ClientCacheFlagsCodecTest.java
index 22e9a25..a9bda0a 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/client/impl/ClientCacheFlagsCodecTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/impl/ClientCacheFlagsCodecTest.java
@@ -18,16 +18,14 @@
package org.apache.ignite.internal.client.impl;
import java.util.Arrays;
-import java.util.Collections;
import java.util.EnumSet;
+import java.util.Set;
+
import junit.framework.TestCase;
+
import org.apache.ignite.internal.client.GridClientCacheFlag;
-import org.apache.ignite.internal.client.impl.connection.GridClientConnection;
-import org.apache.ignite.internal.processors.rest.handlers.cache.GridCacheCommandHandler;
import org.apache.ignite.internal.util.typedef.F;
-import static org.apache.ignite.internal.client.GridClientCacheFlag.KEEP_BINARIES;
-
/**
* Tests conversions between GridClientCacheFlag.
*/
@@ -37,16 +35,13 @@ public class ClientCacheFlagsCodecTest extends TestCase {
*/
public void testEncodingDecodingFullness() {
for (GridClientCacheFlag f : GridClientCacheFlag.values()) {
- if (f == KEEP_BINARIES)
- continue;
-
- int bits = GridClientConnection.encodeCacheFlags(Collections.singleton(f));
+ int bits = GridClientCacheFlag.encodeCacheFlags(EnumSet.of(f));
assertTrue(bits != 0);
- boolean out = GridCacheCommandHandler.parseCacheFlags(bits);
+ Set<GridClientCacheFlag> out = GridClientCacheFlag.parseCacheFlags(bits);
- assertEquals(out, true);
+ assertTrue(out.contains(f));
}
}
@@ -54,9 +49,10 @@ public class ClientCacheFlagsCodecTest extends TestCase {
* Tests that groups of client flags can be correctly converted to corresponding server flag groups.
*/
public void testGroupEncodingDecoding() {
- // all
+ // All.
doTestGroup(GridClientCacheFlag.values());
- // none
+
+ // None.
doTestGroup();
}
@@ -64,15 +60,14 @@ public class ClientCacheFlagsCodecTest extends TestCase {
* @param flags Client flags to be encoded, decoded and checked.
*/
private void doTestGroup(GridClientCacheFlag... flags) {
- EnumSet<GridClientCacheFlag> flagSet = F.isEmpty(flags) ? EnumSet.noneOf(GridClientCacheFlag.class) :
- EnumSet.copyOf(Arrays.asList(flags));
-
- int bits = GridClientConnection.encodeCacheFlags(flagSet);
+ EnumSet<GridClientCacheFlag> flagSet = F.isEmpty(flags)
+ ? EnumSet.noneOf(GridClientCacheFlag.class)
+ : EnumSet.copyOf(Arrays.asList(flags));
- boolean out = GridCacheCommandHandler.parseCacheFlags(bits);
+ int bits = GridClientCacheFlag.encodeCacheFlags(flagSet);
- int length = flagSet.contains(KEEP_BINARIES) ? flagSet.size() - 1 : flagSet.size();
+ Set<GridClientCacheFlag> out = GridClientCacheFlag.parseCacheFlags(bits);
- assertEquals(length > 0, out);
+ assertTrue(out.containsAll(flagSet));
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/core/src/main/java/org/apache/ignite/internal/client/GridClientCacheFlag.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/GridClientCacheFlag.java b/modules/core/src/main/java/org/apache/ignite/internal/client/GridClientCacheFlag.java
index 2e4831e..2e26dd8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/GridClientCacheFlag.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/GridClientCacheFlag.java
@@ -18,6 +18,8 @@
package org.apache.ignite.internal.client;
import java.util.Collection;
+import java.util.EnumSet;
+import java.util.Set;
/**
* Cache projection flags that specify projection behaviour.
@@ -38,6 +40,12 @@ public enum GridClientCacheFlag {
KEEP_BINARIES;
/** */
+ public static final int SKIP_STORE_MASK = 0b1;
+
+ /** */
+ public static final int KEEP_BINARIES_MASK = 0b10;
+
+ /** */
private static final GridClientCacheFlag[] VALS = values();
/**
@@ -49,4 +57,44 @@ public enum GridClientCacheFlag {
public static GridClientCacheFlag fromOrdinal(int ord) {
return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
}
+
+ /**
+ * Encodes cache flags to bit map.
+ *
+ * @param flagSet Set of flags to be encoded.
+ * @return Bit map.
+ */
+ public static int encodeCacheFlags(Collection<GridClientCacheFlag> flagSet) {
+ int bits = 0;
+
+ if (flagSet.contains(SKIP_STORE))
+ bits |= SKIP_STORE_MASK;
+
+ if (flagSet.contains(KEEP_BINARIES))
+ bits |= KEEP_BINARIES_MASK;
+
+ return bits;
+ }
+
+ /**
+ * Retrieves cache flags from corresponding bits.
+ *
+ * @param cacheFlagsBits Integer representation of cache flags bit set.
+ * @return Cache flags.
+ */
+ public static Set<GridClientCacheFlag> parseCacheFlags(int cacheFlagsBits) {
+ boolean skipStore = (cacheFlagsBits & SKIP_STORE_MASK) != 0;
+ boolean keepBinaries = (cacheFlagsBits & KEEP_BINARIES_MASK) != 0;
+
+ if (skipStore & keepBinaries)
+ return EnumSet.of(SKIP_STORE, KEEP_BINARIES);
+
+ if (skipStore)
+ return EnumSet.of(SKIP_STORE);
+
+ if (keepBinaries)
+ return EnumSet.of(KEEP_BINARIES);
+
+ return EnumSet.noneOf(GridClientCacheFlag.class);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnection.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnection.java
index 0d7b9e3..c75bd24 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnection.java
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
import javax.net.ssl.SSLContext;
+
import org.apache.ignite.internal.client.GridClientCacheFlag;
import org.apache.ignite.internal.client.GridClientClosedException;
import org.apache.ignite.internal.client.GridClientDataMetrics;
@@ -106,21 +107,6 @@ public abstract class GridClientConnection {
}
/**
- * Encodes cache flags to bit map.
- *
- * @param flagSet Set of flags to be encoded.
- * @return Bit map.
- */
- public static int encodeCacheFlags(Collection<GridClientCacheFlag> flagSet) {
- int bits = 0;
-
- if (flagSet.contains(GridClientCacheFlag.SKIP_STORE))
- bits |= 1;
-
- return bits;
- }
-
- /**
* Puts key-value pair into cache.
*
* @param cacheName Cache name.
@@ -172,7 +158,7 @@ public abstract class GridClientConnection {
* @param destNodeId Destination node ID.
* @return Whether entry was actually removed.
* @throws GridClientConnectionResetException In case of error.
- * @throws org.apache.ignite.internal.client.GridClientClosedException If client was manually closed before request was sent over network.
+ * @throws GridClientClosedException If client was manually closed before request was sent over network.
*/
public abstract <K> GridClientFutureAdapter<Boolean> cacheRemove(String cacheName, K key,
Set<GridClientCacheFlag> flags, UUID destNodeId)
http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
index e117cc8..6e55df8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
@@ -38,6 +38,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.net.ssl.SSLContext;
+
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.client.GridClientAuthenticationException;
@@ -83,6 +84,7 @@ import org.jetbrains.annotations.Nullable;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.ignite.internal.client.GridClientCacheFlag.KEEP_BINARIES;
+import static org.apache.ignite.internal.client.GridClientCacheFlag.encodeCacheFlags;
import static org.apache.ignite.internal.client.impl.connection.GridClientConnectionCloseReason.CONN_IDLE;
import static org.apache.ignite.internal.client.impl.connection.GridClientConnectionCloseReason.FAILED;
import static org.apache.ignite.internal.processors.rest.client.message.GridClientCacheRequest.GridCacheOperation.APPEND;
@@ -97,8 +99,8 @@ import static org.apache.ignite.internal.processors.rest.client.message.GridClie
import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.MARSHALLER;
/**
- * This class performs request to grid over tcp protocol. Serialization is performed with marshaller
- * provided.
+ * This class performs request to grid over TCP protocol.
+ * Serialization is performed with marshaller provided.
*/
public class GridClientNioTcpConnection extends GridClientConnection {
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandler.java
index 0a545b4..b1f49ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandler.java
@@ -51,6 +51,7 @@ import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.client.GridClientCacheFlag;
import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
import org.apache.ignite.internal.processors.cache.CacheConfigurationOverride;
import org.apache.ignite.internal.processors.cache.CacheInvokeEntry;
@@ -83,6 +84,8 @@ import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.cache.CacheMode.REPLICATED;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.internal.GridClosureCallMode.BALANCE;
+import static org.apache.ignite.internal.client.GridClientCacheFlag.KEEP_BINARIES;
+import static org.apache.ignite.internal.client.GridClientCacheFlag.SKIP_STORE;
import static org.apache.ignite.internal.processors.query.QueryUtils.TEMPLATE_PARTITIONED;
import static org.apache.ignite.internal.processors.query.QueryUtils.TEMPLATE_REPLICATED;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.ATOMIC_DECREMENT;
@@ -182,22 +185,6 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
}
/**
- * Retrieves cache flags from corresponding bits.
- *
- * @param cacheFlagsBits Integer representation of cache flags bit set.
- * @return Skip store flag.
- */
- public static boolean parseCacheFlags(int cacheFlagsBits) {
- if (cacheFlagsBits == 0)
- return false;
-
- if ((cacheFlagsBits & 1) != 0)
- return true;
-
- return false;
- }
-
- /**
* Handles append and prepend commands.
*
* @param ctx Kernal context.
@@ -377,7 +364,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
final Object key = req0.key();
- final boolean skipStore = parseCacheFlags(req0.cacheFlags());
+ final Set<GridClientCacheFlag> cacheFlags = GridClientCacheFlag.parseCacheFlags(req0.cacheFlags());
try {
GridRestCommand cmd = req0.command();
@@ -447,70 +434,70 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
}
case CACHE_CONTAINS_KEYS: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new ContainsKeysCommand(getKeys(req0)));
break;
}
case CACHE_CONTAINS_KEY: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new ContainsKeyCommand(key));
break;
}
case CACHE_GET: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new GetCommand(key));
break;
}
case CACHE_GET_AND_PUT: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new GetAndPutCommand(key, getValue(req0)));
break;
}
case CACHE_GET_AND_REPLACE: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new GetAndReplaceCommand(key, getValue(req0)));
break;
}
case CACHE_GET_AND_PUT_IF_ABSENT: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new GetAndPutIfAbsentCommand(key, getValue(req0)));
break;
}
case CACHE_PUT_IF_ABSENT: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new PutIfAbsentCommand(key, ttl, getValue(req0)));
break;
}
case CACHE_GET_ALL: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new GetAllCommand(getKeys(req0)));
break;
}
case CACHE_PUT: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key, new
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key, new
PutCommand(key, ttl, getValue(req0)));
break;
}
case CACHE_ADD: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new AddCommand(key, ttl, getValue(req0)));
break;
@@ -533,35 +520,35 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
// HashMap wrapping for correct serialization
map = new HashMap<>(map);
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new PutAllCommand(map));
break;
}
case CACHE_REMOVE: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new RemoveCommand(key));
break;
}
case CACHE_REMOVE_VALUE: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new RemoveValueCommand(key, getValue(req0)));
break;
}
case CACHE_REPLACE_VALUE: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new ReplaceValueCommand(key, getValue(req0), req0.value2()));
break;
}
case CACHE_GET_AND_REMOVE: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new GetAndRemoveCommand(key));
break;
@@ -573,7 +560,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
// HashSet wrapping for correct serialization
Set<Object> keys = map == null ? null : new HashSet<>(map.keySet());
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new RemoveAllCommand(keys));
break;
@@ -589,7 +576,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
GridCompoundFuture compFut = new GridCompoundFuture();
for (Object cName : cacheNames)
- compFut.add(executeCommand(req.destinationId(), req.clientId(), (String)cName, skipStore, key,
+ compFut.add(executeCommand(req.destinationId(), req.clientId(), (String)cName, cacheFlags, key,
new RemoveAllCommand(null)));
compFut.markInitialized();
@@ -623,7 +610,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
if (val == null)
throw new IgniteCheckedException(GridRestCommandHandlerAdapter.missingParameter("val"));
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new ReplaceCommand(key, ttl, val));
break;
@@ -633,21 +620,21 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
final Object val1 = req0.value();
final Object val2 = req0.value2();
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new CasCommand(val2, val1, key));
break;
}
case CACHE_APPEND: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new AppendCommand(key, req0));
break;
}
case CACHE_PREPEND: {
- fut = executeCommand(req.destinationId(), req.clientId(), cacheName, skipStore, key,
+ fut = executeCommand(req.destinationId(), req.clientId(), cacheName, cacheFlags, key,
new PrependCommand(key, req0));
break;
@@ -680,12 +667,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
return fut;
}
- catch (IgniteException e) {
- U.error(log, "Failed to execute cache command: " + req, e);
-
- return new GridFinishedFuture<>(e);
- }
- catch (IgniteCheckedException e) {
+ catch (IgniteException | IgniteCheckedException e) {
U.error(log, "Failed to execute cache command: " + req, e);
return new GridFinishedFuture<>(e);
@@ -741,7 +723,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
* @param destId Target node Id for the operation. If {@code null} - operation could be executed anywhere.
* @param clientId Client ID.
* @param cacheName Cache name.
- * @param skipStore Skip store.
+ * @param cacheFlags Cache flags.
* @param key Key to set affinity mapping in the response.
* @param op Operation to perform.
* @return Operation result in future.
@@ -751,7 +733,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
@Nullable UUID destId,
UUID clientId,
final String cacheName,
- final boolean skipStore,
+ final Set<GridClientCacheFlag> cacheFlags,
final Object key,
final CacheProjectionCommand op) throws IgniteCheckedException {
@@ -761,8 +743,10 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
if (locExec) {
IgniteInternalCache<?, ?> prj = localCache(cacheName)
.forSubjectId(clientId)
- .setSkipStore(skipStore)
- .keepBinary();
+ .setSkipStore(cacheFlags.contains(SKIP_STORE));
+
+ if (cacheFlags.contains(KEEP_BINARIES))
+ prj = prj.keepBinary();
return op.apply((IgniteInternalCache<Object, Object>)prj, ctx).
chain(resultWrapper((IgniteInternalCache<Object, Object>)prj, key));
@@ -773,7 +757,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
ctx.task().setThreadContext(TC_NO_FAILOVER, true);
return ctx.closure().callAsync(BALANCE,
- new FlaggedCacheOperationCallable(clientId, cacheName, skipStore, op, key),
+ new FlaggedCacheOperationCallable(clientId, cacheName, cacheFlags, op, key),
prj.nodes());
}
}
@@ -902,7 +886,7 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
/** */
private final String cacheName;
/** */
- private final boolean skipStore;
+ private final Set<GridClientCacheFlag> cacheFlags;
/** */
private final CacheProjectionCommand op;
/** */
@@ -916,19 +900,19 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
/**
* @param clientId Client ID.
* @param cacheName Cache name.
- * @param skipStore Skip store.
+ * @param cacheFlags Cache flags.
* @param op Operation.
* @param key Key.
*/
private FlaggedCacheOperationCallable(UUID clientId,
String cacheName,
- boolean skipStore,
+ Set<GridClientCacheFlag> cacheFlags,
CacheProjectionCommand op,
Object key
) {
this.clientId = clientId;
this.cacheName = cacheName;
- this.skipStore = skipStore;
+ this.cacheFlags = cacheFlags;
this.op = op;
this.key = key;
}
@@ -937,8 +921,10 @@ public class GridCacheCommandHandler extends GridRestCommandHandlerAdapter {
@Override public GridRestResponse call() throws Exception {
IgniteInternalCache<?, ?> prj = cache(g, cacheName)
.forSubjectId(clientId)
- .setSkipStore(skipStore)
- .keepBinary();
+ .setSkipStore(cacheFlags.contains(SKIP_STORE));
+
+ if (cacheFlags.contains(KEEP_BINARIES))
+ prj = prj.keepBinary();
// Need to apply both operation and response transformation remotely
// as cache could be inaccessible on local node and
http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandlerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandlerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandlerSelfTest.java
index e34992f..c94ebd8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandlerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rest/handlers/cache/GridCacheCommandHandlerSelfTest.java
@@ -277,10 +277,15 @@ public class GridCacheCommandHandlerSelfTest extends GridCommonAbstractTest {
return fut;
}
+
// Rewriting flagOn result to keep intercepting invocations after it.
- else if ("setSkipStore".equals(mtd.getName()))
+ if ("setSkipStore".equals(mtd.getName()))
+ return proxy;
+
+ if ("forSubjectId".equals(mtd.getName()))
return proxy;
- else if ("forSubjectId".equals(mtd.getName()))
+
+ if ("keepBinary".equals(mtd.getName()))
return proxy;
return mtd.invoke(cache, args);
http://git-wip-us.apache.org/repos/asf/ignite/blob/90c0af88/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
----------------------------------------------------------------------
diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
index 1d80eea..85398cc 100644
--- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
+++ b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
@@ -67,6 +67,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.internal.client.GridClientCacheFlag.KEEP_BINARIES_MASK;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_CONTAINS_KEYS;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_GET_ALL;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_PUT_ALL;
@@ -625,7 +626,8 @@ public class GridJettyRestHandler extends AbstractHandler {
if (val1 != null)
restReq0.value(val1);
- restReq0.cacheFlags(intValue("cacheFlags", params, 0));
+ // Cache operations via REST will use binary objects.
+ restReq0.cacheFlags(intValue("cacheFlags", params, KEEP_BINARIES_MASK));
restReq0.ttl(longValue("exp", params, null));
if (cmd == CACHE_GET_ALL || cmd == CACHE_PUT_ALL || cmd == CACHE_REMOVE_ALL ||
[27/50] [abbrv] ignite git commit: IGNITE-6860 Add logging and
additional info while OptimizedMarshaller fails to serialize or deserialize
object.
Posted by sb...@apache.org.
IGNITE-6860 Add logging and additional info while OptimizedMarshaller fails to serialize or deserialize object.
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c533e2cc
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c533e2cc
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c533e2cc
Branch: refs/heads/ignite-zk
Commit: c533e2ccb8dcbb2a9596befd4c4eda65a2f2fa0d
Parents: 7366809
Author: Ivan Daschinskiy <iv...@gmail.com>
Authored: Wed Mar 14 12:55:00 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Wed Mar 14 12:55:31 2018 +0300
----------------------------------------------------------------------
.../internal/binary/BinaryClassDescriptor.java | 382 ++++++++++---------
.../internal/binary/BinaryFieldAccessor.java | 61 ++-
.../optimized/OptimizedObjectInputStream.java | 103 ++---
.../optimized/OptimizedObjectOutputStream.java | 94 +++--
.../binary/BinaryObjectExceptionSelfTest.java | 41 ++
.../OptimizedObjectStreamSelfTest.java | 94 ++++-
6 files changed, 478 insertions(+), 297 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c533e2cc/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
index 3e75982..bac5d40 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
@@ -530,283 +530,297 @@ public class BinaryClassDescriptor {
* @throws BinaryObjectException In case of error.
*/
void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
- assert obj != null;
- assert writer != null;
- assert mode != BinaryWriteMode.OPTIMIZED : "OptimizedMarshaller should not be used here: " + cls.getName();
+ try {
+ assert obj != null;
+ assert writer != null;
+ assert mode != BinaryWriteMode.OPTIMIZED : "OptimizedMarshaller should not be used here: " + cls.getName();
- writer.typeId(typeId);
+ writer.typeId(typeId);
- switch (mode) {
- case P_BYTE:
- case BYTE:
- writer.writeByteFieldPrimitive((byte) obj);
+ switch (mode) {
+ case P_BYTE:
+ case BYTE:
+ writer.writeByteFieldPrimitive((byte)obj);
- break;
+ break;
- case P_SHORT:
- case SHORT:
- writer.writeShortFieldPrimitive((short)obj);
+ case P_SHORT:
+ case SHORT:
+ writer.writeShortFieldPrimitive((short)obj);
- break;
+ break;
- case P_INT:
- case INT:
- writer.writeIntFieldPrimitive((int) obj);
+ case P_INT:
+ case INT:
+ writer.writeIntFieldPrimitive((int)obj);
- break;
+ break;
- case P_LONG:
- case LONG:
- writer.writeLongFieldPrimitive((long) obj);
+ case P_LONG:
+ case LONG:
+ writer.writeLongFieldPrimitive((long)obj);
- break;
+ break;
- case P_FLOAT:
- case FLOAT:
- writer.writeFloatFieldPrimitive((float) obj);
+ case P_FLOAT:
+ case FLOAT:
+ writer.writeFloatFieldPrimitive((float)obj);
- break;
+ break;
- case P_DOUBLE:
- case DOUBLE:
- writer.writeDoubleFieldPrimitive((double) obj);
+ case P_DOUBLE:
+ case DOUBLE:
+ writer.writeDoubleFieldPrimitive((double)obj);
- break;
+ break;
- case P_CHAR:
- case CHAR:
- writer.writeCharFieldPrimitive((char) obj);
+ case P_CHAR:
+ case CHAR:
+ writer.writeCharFieldPrimitive((char)obj);
- break;
+ break;
- case P_BOOLEAN:
- case BOOLEAN:
- writer.writeBooleanFieldPrimitive((boolean) obj);
+ case P_BOOLEAN:
+ case BOOLEAN:
+ writer.writeBooleanFieldPrimitive((boolean)obj);
- break;
+ break;
- case DECIMAL:
- writer.doWriteDecimal((BigDecimal)obj);
+ case DECIMAL:
+ writer.doWriteDecimal((BigDecimal)obj);
- break;
+ break;
- case STRING:
- writer.doWriteString((String)obj);
+ case STRING:
+ writer.doWriteString((String)obj);
- break;
+ break;
- case UUID:
- writer.doWriteUuid((UUID)obj);
+ case UUID:
+ writer.doWriteUuid((UUID)obj);
- break;
+ break;
- case DATE:
- writer.doWriteDate((Date)obj);
+ case DATE:
+ writer.doWriteDate((Date)obj);
- break;
+ break;
- case TIMESTAMP:
- writer.doWriteTimestamp((Timestamp)obj);
+ case TIMESTAMP:
+ writer.doWriteTimestamp((Timestamp)obj);
- break;
+ break;
- case TIME:
- writer.doWriteTime((Time)obj);
+ case TIME:
+ writer.doWriteTime((Time)obj);
- break;
+ break;
- case BYTE_ARR:
- writer.doWriteByteArray((byte[])obj);
+ case BYTE_ARR:
+ writer.doWriteByteArray((byte[])obj);
- break;
+ break;
- case SHORT_ARR:
- writer.doWriteShortArray((short[]) obj);
+ case SHORT_ARR:
+ writer.doWriteShortArray((short[])obj);
- break;
+ break;
- case INT_ARR:
- writer.doWriteIntArray((int[]) obj);
+ case INT_ARR:
+ writer.doWriteIntArray((int[])obj);
- break;
+ break;
- case LONG_ARR:
- writer.doWriteLongArray((long[]) obj);
+ case LONG_ARR:
+ writer.doWriteLongArray((long[])obj);
- break;
+ break;
- case FLOAT_ARR:
- writer.doWriteFloatArray((float[]) obj);
+ case FLOAT_ARR:
+ writer.doWriteFloatArray((float[])obj);
- break;
+ break;
- case DOUBLE_ARR:
- writer.doWriteDoubleArray((double[]) obj);
+ case DOUBLE_ARR:
+ writer.doWriteDoubleArray((double[])obj);
- break;
+ break;
- case CHAR_ARR:
- writer.doWriteCharArray((char[]) obj);
+ case CHAR_ARR:
+ writer.doWriteCharArray((char[])obj);
- break;
+ break;
- case BOOLEAN_ARR:
- writer.doWriteBooleanArray((boolean[]) obj);
+ case BOOLEAN_ARR:
+ writer.doWriteBooleanArray((boolean[])obj);
- break;
+ break;
- case DECIMAL_ARR:
- writer.doWriteDecimalArray((BigDecimal[]) obj);
+ case DECIMAL_ARR:
+ writer.doWriteDecimalArray((BigDecimal[])obj);
- break;
+ break;
- case STRING_ARR:
- writer.doWriteStringArray((String[]) obj);
+ case STRING_ARR:
+ writer.doWriteStringArray((String[])obj);
- break;
+ break;
- case UUID_ARR:
- writer.doWriteUuidArray((UUID[]) obj);
+ case UUID_ARR:
+ writer.doWriteUuidArray((UUID[])obj);
- break;
+ break;
- case DATE_ARR:
- writer.doWriteDateArray((Date[]) obj);
+ case DATE_ARR:
+ writer.doWriteDateArray((Date[])obj);
- break;
+ break;
- case TIMESTAMP_ARR:
- writer.doWriteTimestampArray((Timestamp[]) obj);
+ case TIMESTAMP_ARR:
+ writer.doWriteTimestampArray((Timestamp[])obj);
- break;
+ break;
- case TIME_ARR:
- writer.doWriteTimeArray((Time[]) obj);
+ case TIME_ARR:
+ writer.doWriteTimeArray((Time[])obj);
- break;
+ break;
- case OBJECT_ARR:
- writer.doWriteObjectArray((Object[])obj);
+ case OBJECT_ARR:
+ writer.doWriteObjectArray((Object[])obj);
- break;
+ break;
- case COL:
- writer.doWriteCollection((Collection<?>)obj);
+ case COL:
+ writer.doWriteCollection((Collection<?>)obj);
- break;
+ break;
- case MAP:
- writer.doWriteMap((Map<?, ?>)obj);
+ case MAP:
+ writer.doWriteMap((Map<?, ?>)obj);
- break;
+ break;
- case ENUM:
- writer.doWriteEnum((Enum<?>)obj);
+ case ENUM:
+ writer.doWriteEnum((Enum<?>)obj);
- break;
+ break;
- case BINARY_ENUM:
- writer.doWriteBinaryEnum((BinaryEnumObjectImpl)obj);
+ case BINARY_ENUM:
+ writer.doWriteBinaryEnum((BinaryEnumObjectImpl)obj);
- break;
+ break;
- case ENUM_ARR:
- writer.doWriteEnumArray((Object[])obj);
+ case ENUM_ARR:
+ writer.doWriteEnumArray((Object[])obj);
- break;
+ break;
- case CLASS:
- writer.doWriteClass((Class)obj);
+ case CLASS:
+ writer.doWriteClass((Class)obj);
- break;
+ break;
- case PROXY:
- writer.doWriteProxy((Proxy)obj, intfs);
+ case PROXY:
+ writer.doWriteProxy((Proxy)obj, intfs);
- break;
+ break;
- case BINARY_OBJ:
- writer.doWriteBinaryObject((BinaryObjectImpl)obj);
+ case BINARY_OBJ:
+ writer.doWriteBinaryObject((BinaryObjectImpl)obj);
- break;
+ break;
- case BINARY:
- if (preWrite(writer, obj)) {
- try {
- if (serializer != null)
- serializer.writeBinary(obj, writer);
- else
- ((Binarylizable)obj).writeBinary(writer);
+ case BINARY:
+ if (preWrite(writer, obj)) {
+ try {
+ if (serializer != null)
+ serializer.writeBinary(obj, writer);
+ else
+ ((Binarylizable)obj).writeBinary(writer);
- postWrite(writer);
+ postWrite(writer);
- // Check whether we need to update metadata.
- // The reason for this check is described in https://issues.apache.org/jira/browse/IGNITE-7138.
- if (obj.getClass() != BinaryMetadata.class && obj.getClass() != BinaryTreeMap.class) {
- int schemaId = writer.schemaId();
+ // Check whether we need to update metadata.
+ // The reason for this check is described in https://issues.apache.org/jira/browse/IGNITE-7138.
+ if (obj.getClass() != BinaryMetadata.class && obj.getClass() != BinaryTreeMap.class) {
+ int schemaId = writer.schemaId();
- if (schemaReg.schema(schemaId) == null) {
- // This is new schema, let's update metadata.
- BinaryMetadataCollector collector =
- new BinaryMetadataCollector(typeId, typeName, mapper);
+ if (schemaReg.schema(schemaId) == null) {
+ // This is new schema, let's update metadata.
+ BinaryMetadataCollector collector =
+ new BinaryMetadataCollector(typeId, typeName, mapper);
- if (serializer != null)
- serializer.writeBinary(obj, collector);
- else
- ((Binarylizable)obj).writeBinary(collector);
+ if (serializer != null)
+ serializer.writeBinary(obj, collector);
+ else
+ ((Binarylizable)obj).writeBinary(collector);
- BinarySchema newSchema = collector.schema();
+ BinarySchema newSchema = collector.schema();
- BinaryMetadata meta = new BinaryMetadata(typeId, typeName, collector.meta(),
- affKeyFieldName, Collections.singleton(newSchema), false, null);
+ BinaryMetadata meta = new BinaryMetadata(typeId, typeName, collector.meta(),
+ affKeyFieldName, Collections.singleton(newSchema), false, null);
- ctx.updateMetadata(typeId, meta);
+ ctx.updateMetadata(typeId, meta);
- schemaReg.addSchema(newSchema.schemaId(), newSchema);
+ schemaReg.addSchema(newSchema.schemaId(), newSchema);
+ }
}
- }
- postWriteHashCode(writer, obj);
- }
- finally {
- writer.popSchema();
+ postWriteHashCode(writer, obj);
+ }
+ finally {
+ writer.popSchema();
+ }
}
- }
- break;
+ break;
- case OBJECT:
- if (userType && !stableSchemaPublished) {
- // Update meta before write object with new schema
- BinaryMetadata meta = new BinaryMetadata(typeId, typeName, stableFieldsMeta,
- affKeyFieldName, Collections.singleton(stableSchema), false, null);
+ case OBJECT:
+ if (userType && !stableSchemaPublished) {
+ // Update meta before write object with new schema
+ BinaryMetadata meta = new BinaryMetadata(typeId, typeName, stableFieldsMeta,
+ affKeyFieldName, Collections.singleton(stableSchema), false, null);
- ctx.updateMetadata(typeId, meta);
+ ctx.updateMetadata(typeId, meta);
- schemaReg.addSchema(stableSchema.schemaId(), stableSchema);
+ schemaReg.addSchema(stableSchema.schemaId(), stableSchema);
- stableSchemaPublished = true;
- }
+ stableSchemaPublished = true;
+ }
- if (preWrite(writer, obj)) {
- try {
- for (BinaryFieldAccessor info : fields)
- info.write(obj, writer);
+ if (preWrite(writer, obj)) {
+ try {
+ for (BinaryFieldAccessor info : fields)
+ info.write(obj, writer);
- writer.schemaId(stableSchema.schemaId());
+ writer.schemaId(stableSchema.schemaId());
- postWrite(writer);
- postWriteHashCode(writer, obj);
- }
- finally {
- writer.popSchema();
+ postWrite(writer);
+ postWriteHashCode(writer, obj);
+ }
+ finally {
+ writer.popSchema();
+ }
}
- }
- break;
+ break;
- default:
- assert false : "Invalid mode: " + mode;
+ default:
+ assert false : "Invalid mode: " + mode;
+ }
+ }
+ catch (Exception e) {
+ String msg;
+
+ if (S.INCLUDE_SENSITIVE && !F.isEmpty(typeName))
+ msg = "Failed to serialize object [typeName=" + typeName + ']';
+ else
+ msg = "Failed to serialize object [typeId=" + typeId + ']';
+
+ U.error(ctx.log(), msg, e);
+
+ throw new BinaryObjectException(msg, e);
}
}
@@ -871,10 +885,16 @@ public class BinaryClassDescriptor {
return res;
}
catch (Exception e) {
+ String msg;
+
if (S.INCLUDE_SENSITIVE && !F.isEmpty(typeName))
- throw new BinaryObjectException("Failed to deserialize object [typeName=" + typeName + ']', e);
+ msg = "Failed to deserialize object [typeName=" + typeName + ']';
else
- throw new BinaryObjectException("Failed to deserialize object [typeId=" + typeId + ']', e);
+ msg = "Failed to deserialize object [typeId=" + typeId + ']';
+
+ U.error(ctx.log(), msg, e);
+
+ throw new BinaryObjectException(msg, e);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c533e2cc/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java
index 567bee1..3277403 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java
@@ -150,7 +150,26 @@ public abstract class BinaryFieldAccessor {
* @param writer Writer.
* @throws BinaryObjectException If failed.
*/
- public abstract void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException;
+ public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+ try {
+ write0(obj, writer);
+ }
+ catch (Exception ex) {
+ if (S.INCLUDE_SENSITIVE && !F.isEmpty(name))
+ throw new BinaryObjectException("Failed to write field [name=" + name + ']', ex);
+ else
+ throw new BinaryObjectException("Failed to write field [id=" + id + ']', ex);
+ }
+ }
+
+ /**
+ * Write field.
+ *
+ * @param obj Object.
+ * @param writer Writer.
+ * @throws BinaryObjectException If failed.
+ */
+ protected abstract void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException;
/**
* Read field.
@@ -215,7 +234,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
writer.writeFieldIdNoSchemaUpdate(id);
byte val = GridUnsafe.getByteField(obj, offset);
@@ -224,7 +243,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
byte val = reader.readByte(id);
GridUnsafe.putByteField(obj, offset, val);
@@ -245,7 +264,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
writer.writeFieldIdNoSchemaUpdate(id);
boolean val = GridUnsafe.getBooleanField(obj, offset);
@@ -254,7 +273,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
boolean val = reader.readBoolean(id);
GridUnsafe.putBooleanField(obj, offset, val);
@@ -275,7 +294,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
writer.writeFieldIdNoSchemaUpdate(id);
short val = GridUnsafe.getShortField(obj, offset);
@@ -284,7 +303,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
short val = reader.readShort(id);
GridUnsafe.putShortField(obj, offset, val);
@@ -305,7 +324,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
writer.writeFieldIdNoSchemaUpdate(id);
char val = GridUnsafe.getCharField(obj, offset);
@@ -314,7 +333,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
char val = reader.readChar(id);
GridUnsafe.putCharField(obj, offset, val);
@@ -335,7 +354,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
writer.writeFieldIdNoSchemaUpdate(id);
int val = GridUnsafe.getIntField(obj, offset);
@@ -344,7 +363,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
int val = reader.readInt(id);
GridUnsafe.putIntField(obj, offset, val);
@@ -365,7 +384,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
writer.writeFieldIdNoSchemaUpdate(id);
long val = GridUnsafe.getLongField(obj, offset);
@@ -374,7 +393,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
long val = reader.readLong(id);
GridUnsafe.putLongField(obj, offset, val);
@@ -395,7 +414,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
writer.writeFieldIdNoSchemaUpdate(id);
float val = GridUnsafe.getFloatField(obj, offset);
@@ -404,7 +423,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
float val = reader.readFloat(id);
GridUnsafe.putFloatField(obj, offset, val);
@@ -425,7 +444,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
writer.writeFieldIdNoSchemaUpdate(id);
double val = GridUnsafe.getDoubleField(obj, offset);
@@ -434,7 +453,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
+ @Override protected void read0(Object obj, BinaryReaderExImpl reader) throws BinaryObjectException {
double val = reader.readDouble(id);
GridUnsafe.putDoubleField(obj, offset, val);
@@ -466,7 +485,7 @@ public abstract class BinaryFieldAccessor {
}
/** {@inheritDoc} */
- @Override public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
+ @Override protected void write0(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
assert obj != null;
assert writer != null;
@@ -483,17 +502,17 @@ public abstract class BinaryFieldAccessor {
switch (mode(val)) {
case BYTE:
- writer.writeByteField((Byte) val);
+ writer.writeByteField((Byte)val);
break;
case SHORT:
- writer.writeShortField((Short) val);
+ writer.writeShortField((Short)val);
break;
case INT:
- writer.writeIntField((Integer) val);
+ writer.writeIntField((Integer)val);
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c533e2cc/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java
index 0bd7cc0..17eba66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java
@@ -342,7 +342,13 @@ class OptimizedObjectInputStream extends ObjectInputStream {
curCls = desc.describedClass();
- return desc.read(this);
+ try {
+ return desc.read(this);
+ }
+ catch (IOException e){
+ throw new IOException("Failed to deserialize object [typeName=" +
+ desc.describedClass().getName() + ']', e);
+ }
default:
SB msg = new SB("Unexpected error occurred during unmarshalling");
@@ -443,76 +449,81 @@ class OptimizedObjectInputStream extends ObjectInputStream {
for (int i = 0; i < fieldOffs.size(); i++) {
OptimizedClassDescriptor.FieldInfo t = fieldOffs.get(i);
- switch ((t.type())) {
- case BYTE:
- byte resByte = readByte();
+ try {
+ switch ((t.type())) {
+ case BYTE:
+ byte resByte = readByte();
- if (t.field() != null)
- setByte(obj, t.offset(), resByte);
+ if (t.field() != null)
+ setByte(obj, t.offset(), resByte);
- break;
+ break;
- case SHORT:
- short resShort = readShort();
+ case SHORT:
+ short resShort = readShort();
- if (t.field() != null)
- setShort(obj, t.offset(), resShort);
+ if (t.field() != null)
+ setShort(obj, t.offset(), resShort);
- break;
+ break;
- case INT:
- int resInt = readInt();
+ case INT:
+ int resInt = readInt();
- if (t.field() != null)
- setInt(obj, t.offset(), resInt);
+ if (t.field() != null)
+ setInt(obj, t.offset(), resInt);
- break;
+ break;
- case LONG:
- long resLong = readLong();
+ case LONG:
+ long resLong = readLong();
- if (t.field() != null)
- setLong(obj, t.offset(), resLong);
+ if (t.field() != null)
+ setLong(obj, t.offset(), resLong);
- break;
+ break;
- case FLOAT:
- float resFloat = readFloat();
+ case FLOAT:
+ float resFloat = readFloat();
- if (t.field() != null)
- setFloat(obj, t.offset(), resFloat);
+ if (t.field() != null)
+ setFloat(obj, t.offset(), resFloat);
- break;
+ break;
- case DOUBLE:
- double resDouble = readDouble();
+ case DOUBLE:
+ double resDouble = readDouble();
- if (t.field() != null)
- setDouble(obj, t.offset(), resDouble);
+ if (t.field() != null)
+ setDouble(obj, t.offset(), resDouble);
- break;
+ break;
- case CHAR:
- char resChar = readChar();
+ case CHAR:
+ char resChar = readChar();
- if (t.field() != null)
- setChar(obj, t.offset(), resChar);
+ if (t.field() != null)
+ setChar(obj, t.offset(), resChar);
- break;
+ break;
- case BOOLEAN:
- boolean resBoolean = readBoolean();
+ case BOOLEAN:
+ boolean resBoolean = readBoolean();
- if (t.field() != null)
- setBoolean(obj, t.offset(), resBoolean);
+ if (t.field() != null)
+ setBoolean(obj, t.offset(), resBoolean);
- break;
+ break;
- case OTHER:
- Object resObject = readObject();
+ case OTHER:
+ Object resObject = readObject();
- if (t.field() != null)
- setObject(obj, t.offset(), resObject);
+ if (t.field() != null)
+ setObject(obj, t.offset(), resObject);
+ }
+ }
+ catch (IOException e) {
+ throw new IOException("Failed to deserialize field [name=" + t.name() + ']', e);
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c533e2cc/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java
index 8a323a7..66da2da 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java
@@ -230,12 +230,19 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
mapper);
}
- if (handle >= 0) {
- writeByte(HANDLE);
- writeInt(handle);
+ try {
+ if (handle >= 0) {
+ writeByte(HANDLE);
+
+ writeInt(handle);
+ }
+ else
+ desc.write(this, obj);
+ }
+ catch (IOException e){
+ throw new IOException("Failed to serialize object [typeName=" +
+ desc.describedClass().getName() + ']', e);
}
- else
- desc.write(this, obj);
}
}
}
@@ -476,58 +483,63 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
for (int i = 0; i < fields.size(); i++) {
OptimizedClassDescriptor.FieldInfo t = fields.get(i);
- switch (t.type()) {
- case BYTE:
- if (t.field() != null)
- writeByte(getByte(obj, t.offset()));
+ try {
+ switch (t.type()) {
+ case BYTE:
+ if (t.field() != null)
+ writeByte(getByte(obj, t.offset()));
- break;
+ break;
- case SHORT:
- if (t.field() != null)
- writeShort(getShort(obj, t.offset()));
+ case SHORT:
+ if (t.field() != null)
+ writeShort(getShort(obj, t.offset()));
- break;
+ break;
- case INT:
- if (t.field() != null)
- writeInt(getInt(obj, t.offset()));
+ case INT:
+ if (t.field() != null)
+ writeInt(getInt(obj, t.offset()));
- break;
+ break;
- case LONG:
- if (t.field() != null)
- writeLong(getLong(obj, t.offset()));
+ case LONG:
+ if (t.field() != null)
+ writeLong(getLong(obj, t.offset()));
- break;
+ break;
- case FLOAT:
- if (t.field() != null)
- writeFloat(getFloat(obj, t.offset()));
+ case FLOAT:
+ if (t.field() != null)
+ writeFloat(getFloat(obj, t.offset()));
- break;
+ break;
- case DOUBLE:
- if (t.field() != null)
- writeDouble(getDouble(obj, t.offset()));
+ case DOUBLE:
+ if (t.field() != null)
+ writeDouble(getDouble(obj, t.offset()));
- break;
+ break;
- case CHAR:
- if (t.field() != null)
- writeChar(getChar(obj, t.offset()));
+ case CHAR:
+ if (t.field() != null)
+ writeChar(getChar(obj, t.offset()));
- break;
+ break;
- case BOOLEAN:
- if (t.field() != null)
- writeBoolean(getBoolean(obj, t.offset()));
+ case BOOLEAN:
+ if (t.field() != null)
+ writeBoolean(getBoolean(obj, t.offset()));
- break;
+ break;
- case OTHER:
- if (t.field() != null)
- writeObject0(getObject(obj, t.offset()));
+ case OTHER:
+ if (t.field() != null)
+ writeObject0(getObject(obj, t.offset()));
+ }
+ }
+ catch (IOException e) {
+ throw new IOException("Failed to serialize field [name=" + t.name() + ']', e);
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c533e2cc/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectExceptionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectExceptionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectExceptionSelfTest.java
index a3ee93a..0a10fe7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectExceptionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectExceptionSelfTest.java
@@ -24,6 +24,9 @@ import java.util.UUID;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.binary.BinaryBasicNameMapper;
import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
import org.apache.ignite.configuration.BinaryConfiguration;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
@@ -149,6 +152,44 @@ public class BinaryObjectExceptionSelfTest extends GridCommonAbstractTest {
assertEquals("Fields count must match \"Unexpected field type\" exception count", fields.length, unexpectedCnt);
}
+ /**
+ * Test verbose logging of object marshalling.
+ *
+ * @throws Exception If failed.
+ */
+ public void testFailedMarshallingLogging() throws Exception {
+ BinaryMarshaller marshaller = createStandaloneBinaryMarshaller();
+
+ try {
+ marshaller.marshal(new Value1());
+ }
+ catch (BinaryObjectException ex) {
+ assertTrue(ex.getMessage().contains(
+ "object [typeName=org.apache.ignite.internal.binary.BinaryObjectExceptionSelfTest$Value1"));
+
+ assertTrue(ex.getCause().getMessage().contains("field [name=objVal"));
+ }
+ }
+
+ /** */
+ private static class Value1{
+ /** */
+ Value2 objVal = new Value2();
+
+ /** */
+ static class Value2 implements Binarylizable {
+ /** */
+ @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException {
+ throw new RuntimeException("bad object");
+ }
+
+ /** */
+ @Override public void readBinary(BinaryReader reader) throws BinaryObjectException {
+ throw new RuntimeException("bad object");
+ }
+ }
+ }
+
/** */
private enum EnumValues {
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/c533e2cc/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectStreamSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectStreamSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectStreamSelfTest.java
index ed0862b..5efc5b9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectStreamSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectStreamSelfTest.java
@@ -56,6 +56,7 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.Vector;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.util.io.GridUnsafeDataInput;
@@ -67,7 +68,6 @@ import org.apache.ignite.marshaller.MarshallerExclusions;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.jetbrains.annotations.Nullable;
-import java.util.concurrent.ConcurrentHashMap;
import static org.junit.Assert.assertArrayEquals;
@@ -146,13 +146,9 @@ public class OptimizedObjectStreamSelfTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
public void testBoolean() throws Exception {
- boolean val = true;
-
- assertEquals(new Boolean(val), marshalUnmarshal(val));
-
- val = false;
+ assertEquals(Boolean.TRUE, marshalUnmarshal(Boolean.TRUE));
- assertEquals(new Boolean(val), marshalUnmarshal(val));
+ assertEquals(Boolean.FALSE, marshalUnmarshal(Boolean.FALSE));
}
/**
@@ -275,6 +271,49 @@ public class OptimizedObjectStreamSelfTest extends GridCommonAbstractTest {
}
/**
+ * Test informative exception message while failed object unmarshalling
+ *
+ * @throws Exception If failed.
+ */
+ public void testFailedUnmarshallingLogging() throws Exception {
+ OptimizedMarshaller marsh = new OptimizedMarshaller(true);
+
+ marsh.setContext(CTX);
+
+ try {
+ marsh.unmarshal(marsh.marshal(new BadDeserializableObject()), null);
+ }
+ catch (IgniteCheckedException ex) {
+ assertTrue(ex.getCause().getMessage().contains(
+ "object [typeName=org.apache.ignite.internal.marshaller.optimized.OptimizedObjectStreamSelfTest$BadDeserializableObject]"));
+
+ assertTrue(ex.getCause().getCause().getMessage().contains("field [name=val"));
+ }
+ }
+
+
+ /**
+ * Test informative exception message while failed object marshalling
+ *
+ * @throws Exception If failed.
+ */
+ public void testFailedMarshallingLogging() throws Exception {
+ OptimizedMarshaller marsh = new OptimizedMarshaller(true);
+
+ marsh.setContext(CTX);
+
+ try {
+ marsh.marshal(new BadSerializableObject());
+ }
+ catch (IgniteCheckedException ex) {
+ assertTrue(ex.getCause().getMessage().contains(
+ "object [typeName=org.apache.ignite.internal.marshaller.optimized.OptimizedObjectStreamSelfTest$BadSerializableObject]"));
+
+ assertTrue(ex.getCause().getCause().getMessage().contains("field [name=val"));
+ }
+ }
+
+ /**
* @throws Exception If failed.
*/
public void testPool() throws Exception {
@@ -1028,7 +1067,7 @@ public class OptimizedObjectStreamSelfTest extends GridCommonAbstractTest {
marshalUnmarshal(new CustomWriteObjectMethodObject("test"));
}
catch (IOException e) {
- assert e.getCause() instanceof NotActiveException;
+ assert e.getCause().getCause() instanceof NotActiveException;
}
}
@@ -2249,4 +2288,43 @@ public class OptimizedObjectStreamSelfTest extends GridCommonAbstractTest {
return res;
}
}
+
+ /** */
+ static class BadDeserializableObject implements Serializable {
+ /** */
+ BadDeserializableValue val = new BadDeserializableValue();
+ }
+
+ /** */
+ static class BadDeserializableValue implements Serializable {
+ /** */
+ private void writeObject(ObjectOutputStream os) throws IOException{
+ os.write(10);
+ }
+
+ /** */
+ private void readObject(ObjectInputStream os){
+ throw new RuntimeException("bad object");
+ }
+ }
+
+ /** */
+ static class BadSerializableObject implements Serializable {
+ /** */
+ BadSerializableValue val = new BadSerializableValue();
+ }
+
+ /** */
+ static class BadSerializableValue implements Serializable {
+ /** */
+ private void writeObject(ObjectOutputStream os){
+ throw new RuntimeException("bad object");
+ }
+
+ /** */
+ private void readObject(ObjectInputStream os){
+ throw new RuntimeException("bad object");
+ }
+ }
+
}
\ No newline at end of file
[42/50] [abbrv] ignite git commit: IGNITE-7879: Fixed splitter logic
for DISTINCT with subqueries. This closes #3634.
Posted by sb...@apache.org.
IGNITE-7879: Fixed splitter logic for DISTINCT with subqueries. This closes #3634.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/78e8627a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/78e8627a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/78e8627a
Branch: refs/heads/ignite-zk
Commit: 78e8627adae904774aa8513adfbedd0b225cc86e
Parents: f623ef1
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Fri Mar 16 14:38:38 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Mar 16 14:38:38 2018 +0300
----------------------------------------------------------------------
.../query/h2/sql/GridSqlQuerySplitter.java | 3 +-
.../query/SqlPushDownFunctionTest.java | 78 ++++++++++++++++++++
.../IgniteCacheQuerySelfTestSuite.java | 2 +
3 files changed, 82 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/78e8627a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index c96b486..d2fb149 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -962,7 +962,8 @@ public class GridSqlQuerySplitter {
select.setColumn(i, expr);
}
- if (isAllRelatedToTables(tblAliases, GridSqlQuerySplitter.<GridSqlAlias>newIdentityHashSet(), expr)) {
+ if (isAllRelatedToTables(tblAliases, GridSqlQuerySplitter.<GridSqlAlias>newIdentityHashSet(), expr)
+ && !hasAggregates(expr)) {
// Push down the whole expression.
pushDownColumn(tblAliases, cols, wrapAlias, expr, 0);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/78e8627a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPushDownFunctionTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPushDownFunctionTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPushDownFunctionTest.java
new file mode 100644
index 0000000..9e78770
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlPushDownFunctionTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import java.util.List;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for schemas.
+ */
+public class SqlPushDownFunctionTest extends GridCommonAbstractTest {
+ /** Node. */
+ private IgniteEx node;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ node = (IgniteEx)startGrid();
+
+ startGrid(2);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ */
+ public void testPushDownFunction() {
+ sql("CREATE TABLE Person(id INTEGER PRIMARY KEY, company_id INTEGER)");
+ sql("CREATE TABLE Company(id INTEGER PRIMARY KEY, name VARCHAR)");
+
+ sql("INSERT INTO Person(id,company_id) VALUES (1, 1), (2, 2), (3, 3)");
+ sql("INSERT INTO Company(id,name) VALUES (1,'n1'), (2,'n2'), (3,'n3')");
+
+ sql("SELECT p.id, sum(p.id) FROM person p " +
+ "LEFT JOIN (select distinct id from company) as c on c.id=p.company_id " +
+ "GROUP BY p.id");
+
+ sql("SELECT count(1) FROM person p " +
+ "LEFT JOIN (select id from company union select id from company) as c on c.id=p.company_id");
+
+ sql("SELECT count(1) FROM person p " +
+ "LEFT JOIN (select id from company union select id from company) as c on c.id=p.company_id");
+
+ sql("SELECT 1, (select count(1)) FROM person p " +
+ "LEFT JOIN (select id from company union select id from company) as c on c.id=p.company_id");
+ }
+
+ /**
+ * @param sql SQL query.
+ * @return Results.
+ */
+ private List<List<?>> sql(String sql) {
+ GridQueryProcessor qryProc = node.context().query();
+
+ SqlFieldsQuery qry = new SqlFieldsQuery(sql).setSchema("PUBLIC");
+
+ return qryProc.querySqlFields(qry, true).getAll();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/78e8627a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index c01eaa6..35d1be2 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -157,6 +157,7 @@ import org.apache.ignite.internal.processors.query.IgniteSqlSkipReducerOnUpdateD
import org.apache.ignite.internal.processors.query.IgniteSqlSplitterSelfTest;
import org.apache.ignite.internal.processors.query.LazyQuerySelfTest;
import org.apache.ignite.internal.processors.query.MultipleStatementsSqlQuerySelfTest;
+import org.apache.ignite.internal.processors.query.SqlPushDownFunctionTest;
import org.apache.ignite.internal.processors.query.SqlSchemaSelfTest;
import org.apache.ignite.internal.processors.query.h2.GridH2IndexingInMemSelfTest;
import org.apache.ignite.internal.processors.query.h2.GridH2IndexingOffheapSelfTest;
@@ -230,6 +231,7 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
// Queries tests.
suite.addTestSuite(LazyQuerySelfTest.class);
suite.addTestSuite(IgniteSqlSplitterSelfTest.class);
+ suite.addTestSuite(SqlPushDownFunctionTest.class);
suite.addTestSuite(IgniteSqlSegmentedIndexSelfTest.class);
suite.addTestSuite(IgniteCachelessQueriesSelfTest.class);
suite.addTestSuite(IgniteSqlSegmentedIndexMultiNodeSelfTest.class);
[12/50] [abbrv] ignite git commit: IGNITE-7876: Adopt SVM Linear
Binary Classification Model and Trainer to the new Partitioned Dataset
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/LocalLinearSVMBinaryClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/LocalLinearSVMBinaryClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/LocalLinearSVMBinaryClassificationTrainerTest.java
deleted file mode 100644
index a9cb54c..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/binary/LocalLinearSVMBinaryClassificationTrainerTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm.binary;
-
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
-import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationTrainer;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link DenseLocalOnHeapMatrix}.
- */
-public class LocalLinearSVMBinaryClassificationTrainerTest extends GenericLinearSVMBinaryClassificationTrainerTest {
- /** */
- public LocalLinearSVMBinaryClassificationTrainerTest() {
- super(
- new SVMLinearBinaryClassificationTrainer()
- .withLambda(0.2)
- .withAmountOfIterations(10)
- .withAmountOfLocIterations(20),
- false,
- 1e-2);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/DistributedLinearSVMMultiClassClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/DistributedLinearSVMMultiClassClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/DistributedLinearSVMMultiClassClassificationTrainerTest.java
deleted file mode 100644
index 6806e0b..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/DistributedLinearSVMMultiClassClassificationTrainerTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm.multi;
-
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
-import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationTrainer;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link DenseLocalOnHeapMatrix}.
- */
-public class DistributedLinearSVMMultiClassClassificationTrainerTest extends GenericLinearSVMMultiClassClassificationTrainerTest {
- /** */
- public DistributedLinearSVMMultiClassClassificationTrainerTest() {
- super(
- new SVMLinearMultiClassClassificationTrainer(),
- true,
- 1e-2);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/GenericLinearSVMMultiClassClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/GenericLinearSVMMultiClassClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/GenericLinearSVMMultiClassClassificationTrainerTest.java
deleted file mode 100644
index 8c6083d..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/GenericLinearSVMMultiClassClassificationTrainerTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm.multi;
-
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledDataset;
-import org.apache.ignite.ml.svm.BaseSVMTest;
-import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationModel;
-import org.junit.Test;
-
-/**
- * Base class for all linear regression trainers.
- */
-public class GenericLinearSVMMultiClassClassificationTrainerTest extends BaseSVMTest {
- /** */
- private final Trainer<SVMLinearMultiClassClassificationModel, LabeledDataset> trainer;
-
- /** */
- private boolean isDistributed;
-
- /** */
- private final double precision;
-
- /** */
- GenericLinearSVMMultiClassClassificationTrainerTest(
- Trainer<SVMLinearMultiClassClassificationModel, LabeledDataset> trainer,
- boolean isDistributed,
- double precision) {
- super();
- this.trainer = trainer;
- this.precision = precision;
- this.isDistributed = isDistributed;
- }
-
- /**
- * Test trainer on classification model y = x.
- */
- @Test
- public void testTrainWithTheLinearlySeparableCase() {
- if (isDistributed)
- IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
- double[][] mtx =
- new double[][] {
- {-10.0, 12.0},
- {-5.0, 14.0},
- {-3.0, 18.0},
- {13.0, -1.0},
- {10.0, -2.0},
- {15.0, -3.0}};
- double[] lbs = new double[] {1.0, 1.0, 1.0, 2.0, 2.0, 2.0};
- LabeledDataset dataset = new LabeledDataset(mtx, lbs, null, isDistributed);
-
-
- SVMLinearMultiClassClassificationModel mdl = trainer.train(dataset);
- TestUtils.assertEquals(1.0, mdl.apply(new DenseLocalOnHeapVector(new double[] {-2.0, 15})), precision);
- TestUtils.assertEquals(2.0, mdl.apply(new DenseLocalOnHeapVector(new double[] {12, -5})), precision);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318ffe50/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/LocalLinearSVMMultiClassClassificationTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/LocalLinearSVMMultiClassClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/LocalLinearSVMMultiClassClassificationTrainerTest.java
deleted file mode 100644
index a239c95..0000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/multi/LocalLinearSVMMultiClassClassificationTrainerTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm.multi;
-
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
-import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationTrainer;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link DenseLocalOnHeapMatrix}.
- */
-public class LocalLinearSVMMultiClassClassificationTrainerTest extends GenericLinearSVMMultiClassClassificationTrainerTest {
- /** */
- public LocalLinearSVMMultiClassClassificationTrainerTest() {
- super(
- new SVMLinearMultiClassClassificationTrainer()
- .withLambda(0.2)
- .withAmountOfIterations(10)
- .withAmountOfLocIterations(20),
- false,
- 1e-2);
- }
-}
[07/50] [abbrv] ignite git commit: IGNITE-7880 Web Console: Return
enum values from SQL queries.
Posted by sb...@apache.org.
IGNITE-7880 Web Console: Return enum values from SQL queries.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a8170f78
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a8170f78
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a8170f78
Branch: refs/heads/ignite-zk
Commit: a8170f78a384af024bdd483294f2f241bef5a7f8
Parents: caba7cc
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Wed Mar 7 18:33:28 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Mar 7 18:33:28 2018 +0700
----------------------------------------------------------------------
.../apache/ignite/internal/visor/query/VisorQueryUtils.java | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a8170f78/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java
index aa4cb48..a47acf6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryUtils.java
@@ -195,6 +195,15 @@ public class VisorQueryUtils {
}
if (meta != null) {
+ if (meta.isEnum()) {
+ try {
+ return obj.deserialize().toString();
+ }
+ catch (BinaryObjectException ignore) {
+ // NO-op.
+ }
+ }
+
SB buf = new SB(meta.typeName());
if (meta.fieldNames() != null) {
[23/50] [abbrv] ignite git commit: IGNITE-7895 Web Console: Replaced
PhantomJS with headless Chrome. Cleanup dev dependencies. Added docker image
to run tests on TeamCity.
Posted by sb...@apache.org.
IGNITE-7895 Web Console: Replaced PhantomJS with headless Chrome. Cleanup dev dependencies. Added docker image to run tests on TeamCity.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/536e8254
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/536e8254
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/536e8254
Branch: refs/heads/ignite-zk
Commit: 536e825483deec864ee665d53cc8a695771c4aa0
Parents: 05d58bb
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Wed Mar 14 11:23:29 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Mar 14 11:23:29 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/package.json | 19 ++------
.../web-console/frontend/test/ci/.dockerignore | 4 ++
modules/web-console/frontend/test/ci/Dockerfile | 36 ++++++++++++++
.../frontend/test/ci/docker-compose.yml | 25 ++++++++++
.../frontend/test/karma.conf.babel.js | 12 +++--
.../frontend/test/protractor.conf.js | 50 --------------------
6 files changed, 77 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/536e8254/modules/web-console/frontend/package.json
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 100e644..fd301b2 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -8,8 +8,6 @@
"dev": "npm start",
"build": "webpack --config ./webpack/webpack.prod.babel.js",
"test": "karma start ./test/karma.conf.js",
- "enve2e": "node ./test/e2e/envtools.js start",
- "teste2e": "node ./test/e2e/testcafe.js --env=true",
"eslint": "eslint --format node_modules/eslint-friendly-formatter app/ controllers/ ignite_modules/ -- --eff-by-issue"
},
"author": "",
@@ -106,29 +104,18 @@
},
"devDependencies": {
"chai": "4.1.0",
- "copy-dir": "^0.3.0",
- "fs-extra": "^4.0.2",
- "glob": "^7.1.2",
+ "glob": "7.1.2",
"jasmine-core": "2.6.4",
"karma": "1.7.0",
"karma-babel-preprocessor": "6.0.1",
+ "karma-chrome-launcher": "2.2.0",
"karma-jasmine": "1.1.0",
"karma-mocha": "1.3.0",
"karma-mocha-reporter": "2.2.3",
- "karma-phantomjs-launcher": "1.0.4",
"karma-teamcity-reporter": "1.0.0",
"karma-webpack": "2.0.4",
- "minimist": "^1.2.0",
"mocha": "3.4.2",
- "mocha-teamcity-reporter": "1.1.1",
- "mongodb": "2.2.33",
- "node-cmd": "3.0.0",
- "objectid": "3.2.1",
- "phantomjs-prebuilt": "2.1.14",
"sinon": "2.3.8",
- "testcafe": "0.18.5",
- "testcafe-angular-selectors": "0.3.0",
- "type-detect": "4.0.3",
- "util": "0.10.3"
+ "type-detect": "4.0.3"
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/536e8254/modules/web-console/frontend/test/ci/.dockerignore
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/test/ci/.dockerignore b/modules/web-console/frontend/test/ci/.dockerignore
new file mode 100644
index 0000000..bcac98f
--- /dev/null
+++ b/modules/web-console/frontend/test/ci/.dockerignore
@@ -0,0 +1,4 @@
+.git
+*Dockerfile*
+*docker-compose*
+*/node_modules*
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/536e8254/modules/web-console/frontend/test/ci/Dockerfile
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/test/ci/Dockerfile b/modules/web-console/frontend/test/ci/Dockerfile
new file mode 100644
index 0000000..a1b7c67
--- /dev/null
+++ b/modules/web-console/frontend/test/ci/Dockerfile
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM alpine:edge
+
+RUN apk --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/testing/ add \
+ nodejs nodejs-npm chromium xwininfo xvfb dbus eudev ttf-freefont fluxbox
+
+ENV CHROME_BIN /usr/bin/chromium-browser
+
+# Install frontend & backend apps.
+RUN mkdir -p /opt/web-console
+
+# Copy source.
+WORKDIR /opt/web-console
+COPY . ./frontend
+
+# Install node modules.
+WORKDIR /opt/web-console/frontend
+RUN npm install
+
+ENTRYPOINT ["npm", "test"]
http://git-wip-us.apache.org/repos/asf/ignite/blob/536e8254/modules/web-console/frontend/test/ci/docker-compose.yml
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/test/ci/docker-compose.yml b/modules/web-console/frontend/test/ci/docker-compose.yml
new file mode 100644
index 0000000..69667e6
--- /dev/null
+++ b/modules/web-console/frontend/test/ci/docker-compose.yml
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: '2'
+services:
+ unit_tests:
+ build:
+ context: '../../'
+ dockerfile: './test/ci/Dockerfile'
+ environment:
+ - TEST_REPORTER=teamcity
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/536e8254/modules/web-console/frontend/test/karma.conf.babel.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/test/karma.conf.babel.js b/modules/web-console/frontend/test/karma.conf.babel.js
index 30e168c..48104c8 100644
--- a/modules/web-console/frontend/test/karma.conf.babel.js
+++ b/modules/web-console/frontend/test/karma.conf.babel.js
@@ -35,7 +35,7 @@ export default (config) => {
],
plugins: [
- require('karma-phantomjs-launcher'),
+ require('karma-chrome-launcher'),
require('karma-teamcity-reporter'),
require('karma-mocha-reporter'),
require('karma-webpack'),
@@ -57,7 +57,7 @@ export default (config) => {
// Test results reporter to use
// possible values: 'dots', 'progress'
// available reporters: https://npmjs.org/browse/keyword/karma-reporter.
- reporters: ['mocha'],
+ reporters: [process.env.TEST_REPORTER || 'mocha'],
mochaReporter: {
showDiff: true
@@ -78,7 +78,13 @@ export default (config) => {
// start these browsers
// available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
- browsers: ['PhantomJS'],
+ browsers: ['ChromeHeadlessNoSandbox'],
+ customLaunchers: {
+ ChromeHeadlessNoSandbox: {
+ base: 'ChromeHeadless',
+ flags: ['--no-sandbox']
+ }
+ },
// Continuous Integration mode
// if true, Karma captures browsers, runs the tests and exits
http://git-wip-us.apache.org/repos/asf/ignite/blob/536e8254/modules/web-console/frontend/test/protractor.conf.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/test/protractor.conf.js b/modules/web-console/frontend/test/protractor.conf.js
deleted file mode 100644
index 4ee1c69..0000000
--- a/modules/web-console/frontend/test/protractor.conf.js
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// exports.config = {
-// specs: ['test/e2e/*.js'],
-// capabilities: {
-
-// }
-// };
-
-exports.config = {
- seleniumAddress: 'http://localhost:4444/wd/hub',
-
- capabilities: {
- browserName: 'chrome'
- // 'browserName': 'phantomjs',
-
- // /*
- // * Can be used to specify the phantomjs binary path.
- // * This can generally be ommitted if you installed phantomjs globally.
- // */
- // 'phantomjs.binary.path': require('phantomjs').path,
-
- // /*
- // * Command line args to pass to ghostdriver, phantomjs's browser driver.
- // * See https://github.com/detro/ghostdriver#faq
- // */
- // 'phantomjs.ghostdriver.cli.args': ['--loglevel=DEBUG']
- },
-
- specs: ['test/e2e/*.js'],
-
- jasmineNodeOpts: {
- showColors: true
- }
-};
[35/50] [abbrv] ignite git commit: .NET: Suppress incorrect code
inspection errors
Posted by sb...@apache.org.
.NET: Suppress incorrect code inspection errors
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9fd6f4a4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9fd6f4a4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9fd6f4a4
Branch: refs/heads/ignite-zk
Commit: 9fd6f4a4f64581f3ec4a5a3a7e489226afa1ed0b
Parents: 22eda40
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Thu Mar 15 16:48:46 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Thu Mar 15 16:48:46 2018 +0300
----------------------------------------------------------------------
.../dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs | 1 +
.../platforms/dotnet/Apache.Ignite.Linq/Impl/ExpressionWalker.cs | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9fd6f4a4/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs
index cc12caa..1621183 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/DelegateTypeDescriptor.cs
@@ -203,6 +203,7 @@ namespace Apache.Ignite.Core.Impl.Common
/// Throws an exception if first argument is not null.
/// </summary>
// ReSharper disable once UnusedParameter.Local
+ // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Local
private static void ThrowIfMultipleInterfaces(object check, Type userType, Type interfaceType)
{
if (check != null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/9fd6f4a4/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/ExpressionWalker.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/ExpressionWalker.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/ExpressionWalker.cs
index 36655bc..0d6306a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/ExpressionWalker.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/ExpressionWalker.cs
@@ -119,6 +119,7 @@ namespace Apache.Ignite.Linq.Impl
/// Tries to find QuerySourceReferenceExpression
/// </summary>
public static QuerySourceReferenceExpression GetQuerySourceReference(Expression expression,
+ // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Global
bool throwWhenNotFound = true)
{
var reference = expression as QuerySourceReferenceExpression;