You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/01/17 08:47:13 UTC
[1/3] ignite git commit: IGNITE-4518 Fixed parallel load of cache. -
Fixes #1426.
Repository: ignite
Updated Branches:
refs/heads/ignite-3477 6c31bc1a5 -> e7e0d876b
IGNITE-4518 Fixed parallel load of cache. - Fixes #1426.
Signed-off-by: Andrey Novikov <an...@gridgain.com>
(cherry picked from commit 79401b2)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/393eb940
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/393eb940
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/393eb940
Branch: refs/heads/ignite-3477
Commit: 393eb940eccef149fdd7a5162a533cebd225972e
Parents: 6e5bfea
Author: Andrey Novikov <an...@gridgain.com>
Authored: Mon Jan 16 10:33:16 2017 +0700
Committer: Andrey Novikov <an...@gridgain.com>
Committed: Tue Jan 17 13:45:26 2017 +0700
----------------------------------------------------------------------
.../store/jdbc/dialect/BasicJdbcDialect.java | 31 ++++++++-----
.../store/jdbc/CacheJdbcPojoStoreTest.java | 48 +++++++++++++++++++-
2 files changed, 66 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/393eb940/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/dialect/BasicJdbcDialect.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/dialect/BasicJdbcDialect.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/dialect/BasicJdbcDialect.java
index 3ab112a..139f3fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/dialect/BasicJdbcDialect.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/dialect/BasicJdbcDialect.java
@@ -173,13 +173,15 @@ public class BasicJdbcDialect implements JdbcDialect {
if (appendLowerBound) {
sb.a("(");
- for (int cnt = keyCols.size(); cnt > 0; cnt--) {
- for (int j = 0; j < cnt; j++)
- if (j == cnt - 1)
- sb.a(cols[j]).a(" > ? ");
+ for (int keyCnt = keyCols.size(); keyCnt > 0; keyCnt--) {
+ for (int idx = 0; idx < keyCnt; idx++) {
+ if (idx == keyCnt - 1)
+ sb.a(cols[idx]).a(" > ? ");
else
- sb.a(cols[j]).a(" = ? AND ");
- if (cnt != 1)
+ sb.a(cols[idx]).a(" = ? AND ");
+ }
+
+ if (keyCnt != 1)
sb.a("OR ");
}
@@ -192,13 +194,18 @@ public class BasicJdbcDialect implements JdbcDialect {
if (appendUpperBound) {
sb.a("(");
- for (int cnt = keyCols.size(); cnt > 0; cnt--) {
- for (int j = 0; j < cnt; j++)
- if (j == cnt - 1)
- sb.a(cols[j]).a(" <= ? ");
+ for (int keyCnt = keyCols.size(); keyCnt > 0; keyCnt--) {
+ for (int idx = 0, lastIdx = keyCnt - 1; idx < keyCnt; idx++) {
+ sb.a(cols[idx]);
+
+ // For composite key when not all of the key columns are constrained should use < (strictly less).
+ if (idx == lastIdx)
+ sb.a(keyCnt == keyCols.size() ? " <= ? " : " < ? ");
else
- sb.a(cols[j]).a(" = ? AND ");
- if (cnt != 1)
+ sb.a(" = ? AND ");
+ }
+
+ if (keyCnt != 1)
sb.a(" OR ");
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/393eb940/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
index d8f75d3..4a0b1da 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreTest.java
@@ -216,7 +216,7 @@ public class CacheJdbcPojoStoreTest extends GridAbstractCacheStoreSelfTest<Cache
stmt.executeUpdate("CREATE TABLE IF NOT EXISTS " +
"Person_Complex (id integer not null, org_id integer not null, city_id integer not null, " +
- "name varchar(50), salary integer, PRIMARY KEY(id))");
+ "name varchar(50), salary integer, PRIMARY KEY(id, org_id, city_id))");
conn.commit();
@@ -352,6 +352,52 @@ public class CacheJdbcPojoStoreTest extends GridAbstractCacheStoreSelfTest<Cache
/**
* @throws Exception If failed.
*/
+ public void testParallelLoad() throws Exception {
+ Connection conn = store.openConnection(false);
+
+ PreparedStatement prnComplexStmt = conn.prepareStatement("INSERT INTO Person_Complex(id, org_id, city_id, name, salary) VALUES (?, ?, ?, ?, ?)");
+
+ for (int i = 0; i < 8; i++) {
+
+ prnComplexStmt.setInt(1, (i >> 2) & 1);
+ prnComplexStmt.setInt(2, (i >> 1) & 1);
+ prnComplexStmt.setInt(3, i % 2);
+
+ prnComplexStmt.setString(4, "name");
+ prnComplexStmt.setInt(5, 1000 + i * 500);
+
+ prnComplexStmt.addBatch();
+ }
+
+ prnComplexStmt.executeBatch();
+
+ U.closeQuiet(prnComplexStmt);
+
+ conn.commit();
+
+ U.closeQuiet(conn);
+
+ final Collection<PersonComplexKey> prnComplexKeys = new ConcurrentLinkedQueue<>();
+
+ IgniteBiInClosure<Object, Object> c = new CI2<Object, Object>() {
+ @Override public void apply(Object k, Object v) {
+ if (k instanceof PersonComplexKey && v instanceof Person)
+ prnComplexKeys.add((PersonComplexKey)k);
+ else
+ fail("Unexpected entry [key=" + k + ", value=" + v + "]");
+ }
+ };
+
+ store.setParallelLoadCacheMinimumThreshold(2);
+
+ store.loadCache(c);
+
+ assertEquals(8, prnComplexKeys.size());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testWriteRetry() throws Exception {
CacheJdbcPojoStore<Object, Object> store = store();
[2/3] ignite git commit: IGNITE-4545 Added cache for router
hostnames. - Fixes #1428.
Posted by sb...@apache.org.
IGNITE-4545 Added cache for router hostnames. - Fixes #1428.
Signed-off-by: Andrey Novikov <an...@gridgain.com>
(cherry picked from commit 27ba69b)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b0ac987d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b0ac987d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b0ac987d
Branch: refs/heads/ignite-3477
Commit: b0ac987d3202f5796c101bfc1998e4d4dcfb2bab
Parents: 393eb94
Author: Andrey Novikov <an...@gridgain.com>
Authored: Mon Jan 16 11:22:34 2017 +0700
Committer: Andrey Novikov <an...@gridgain.com>
Committed: Tue Jan 17 13:45:33 2017 +0700
----------------------------------------------------------------------
.../GridClientConnectionManagerAdapter.java | 7 ++-
.../impl/connection/GridClientTopology.java | 53 ++++++++++++++++----
2 files changed, 49 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b0ac987d/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
index 12baee0..f714e7a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
@@ -85,6 +85,9 @@ public abstract class GridClientConnectionManagerAdapter implements GridClientCo
/** Class logger. */
private final Logger log;
+ /** All local enabled MACs. */
+ private final Collection<String> macs;
+
/** NIO server. */
private GridNioServer srv;
@@ -166,6 +169,8 @@ public abstract class GridClientConnectionManagerAdapter implements GridClientCo
if (marshId == null && cfg.getMarshaller() == null)
throw new GridClientException("Failed to start client (marshaller is not configured).");
+ macs = U.allLocalMACs();
+
if (cfg.getProtocol() == GridClientProtocol.TCP) {
try {
IgniteLogger gridLog = new JavaLogger(false);
@@ -316,7 +321,7 @@ public abstract class GridClientConnectionManagerAdapter implements GridClientCo
}
boolean sameHost = node.attributes().isEmpty() ||
- F.containsAny(U.allLocalMACs(), node.attribute(ATTR_MACS).toString().split(", "));
+ F.containsAny(macs, node.attribute(ATTR_MACS).toString().split(", "));
Collection<InetSocketAddress> srvs = new LinkedHashSet<>();
http://git-wip-us.apache.org/repos/asf/ignite/blob/b0ac987d/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientTopology.java
index effd5b3..97aa586 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientTopology.java
@@ -21,7 +21,6 @@ import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.Map;
import java.util.Set;
@@ -61,12 +60,18 @@ public class GridClientTopology {
/** Cached last error prevented topology from update. */
private GridClientException lastError;
+ /** Router addresses from configuration. */
+ private final String routers;
+
/**
* Set of router addresses to infer direct connectivity
* when client is working in router connection mode.
* {@code null} when client is working in direct connection node.
*/
- private final Set<String> routerAddrs;
+ private final Set<InetSocketAddress> routerAddrs;
+
+ /** List of all known local MACs */
+ private final Collection<String> macsCache;
/** Protocol. */
private final GridClientProtocol prot;
@@ -96,8 +101,38 @@ public class GridClientTopology {
metricsCache = cfg.isEnableMetricsCache();
attrCache = cfg.isEnableAttributesCache();
prot = cfg.getProtocol();
- routerAddrs = (!cfg.getRouters().isEmpty() && cfg.getServers().isEmpty()) ?
- new HashSet<>(cfg.getRouters()) : null;
+
+ if (!cfg.getRouters().isEmpty() && cfg.getServers().isEmpty()) {
+ routers = cfg.getRouters().toString();
+
+ routerAddrs = U.newHashSet(cfg.getRouters().size());
+
+ for (String router : cfg.getRouters()) {
+ int portIdx = router.lastIndexOf(":");
+
+ if (portIdx > 0) {
+ String hostName = router.substring(0, portIdx);
+
+ try {
+ int port = Integer.parseInt(router.substring(portIdx + 1));
+
+ InetSocketAddress inetSockAddr = new InetSocketAddress(hostName, port);
+
+ routerAddrs.add(inetSockAddr);
+ }
+ catch (Exception ignore) {
+ // No-op.
+ }
+ }
+ }
+ }
+ else {
+ routers = null;
+
+ routerAddrs = Collections.emptySet();
+ }
+
+ macsCache = U.allLocalMACs();
}
/**
@@ -279,7 +314,7 @@ public class GridClientTopology {
try {
if (lastError != null)
throw new GridClientDisconnectedException(
- "Topology is failed [protocol=" + prot + ", routers=" + routerAddrs + ']', lastError);
+ "Topology is failed [protocol=" + prot + ", routers=" + routers + ']', lastError);
else
return nodes.get(id);
}
@@ -376,19 +411,17 @@ public class GridClientTopology {
(metricsCache && attrCache) || (node.attributes().isEmpty() && node.metrics() == null);
// Try to bypass object copying.
- if (noAttrsAndMetrics && routerAddrs == null && node.connectable())
+ if (noAttrsAndMetrics && routerAddrs.isEmpty() && node.connectable())
return node;
// Return a new node instance based on the original one.
GridClientNodeImpl.Builder nodeBuilder = GridClientNodeImpl.builder(node, !attrCache, !metricsCache);
for (InetSocketAddress addr : node.availableAddresses(prot, true)) {
- boolean router = routerAddrs == null ||
- routerAddrs.contains(addr.getHostName() + ":" + addr.getPort()) ||
- routerAddrs.contains(addr.getAddress().getHostAddress() + ":" + addr.getPort());
+ boolean router = routerAddrs.isEmpty() || routerAddrs.contains(addr);
boolean reachable = noAttrsAndMetrics || !addr.getAddress().isLoopbackAddress() ||
- F.containsAny(U.allLocalMACs(), node.attribute(ATTR_MACS).toString().split(", "));
+ F.containsAny(macsCache, node.<String>attribute(ATTR_MACS).split(", "));
if (router && reachable) {
nodeBuilder.connectable(true);
[3/3] ignite git commit: Merge remote-tracking branch
'remotes/community/ignite-gg-8.0.2.ea2' into ignite-3477
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/community/ignite-gg-8.0.2.ea2' into ignite-3477
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e7e0d876
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e7e0d876
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e7e0d876
Branch: refs/heads/ignite-3477
Commit: e7e0d876b609f44443b2dabc8109471696e940ba
Parents: 6c31bc1 b0ac987
Author: sboikov <sb...@gridgain.com>
Authored: Tue Jan 17 11:45:58 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Jan 17 11:45:58 2017 +0300
----------------------------------------------------------------------
.../store/jdbc/dialect/BasicJdbcDialect.java | 31 +++++++-----
.../GridClientConnectionManagerAdapter.java | 7 ++-
.../impl/connection/GridClientTopology.java | 53 ++++++++++++++++----
.../store/jdbc/CacheJdbcPojoStoreTest.java | 48 +++++++++++++++++-
4 files changed, 115 insertions(+), 24 deletions(-)
----------------------------------------------------------------------