You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by se...@apache.org on 2017/06/08 17:40:51 UTC
[1/2] ignite git commit: master - fixes for push down with LEFT OUTER
JOIN
Repository: ignite
Updated Branches:
refs/heads/master a1e6edcf4 -> 4b0998e5d
master - fixes for push down with LEFT OUTER JOIN
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e3ff16ab
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e3ff16ab
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e3ff16ab
Branch: refs/heads/master
Commit: e3ff16abed72e2b8a897a2699fe23e0e614f502d
Parents: d2efe78
Author: Sergi Vladykin <se...@gmail.com>
Authored: Thu Jun 8 20:39:05 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Thu Jun 8 20:39:05 2017 +0300
----------------------------------------------------------------------
.../processors/query/h2/sql/GridSqlJoin.java | 7 ++
.../query/h2/sql/GridSqlQuerySplitter.java | 82 ++++++++++++++++++--
.../query/IgniteSqlSplitterSelfTest.java | 50 ++++++++++++
3 files changed, 131 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3ff16ab/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java
index afe4a53..9546223 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java
@@ -93,6 +93,13 @@ public class GridSqlJoin extends GridSqlElement {
return child(ON_CHILD);
}
+ /**
+ * @return {@code true} If this is a LEFT OUTER JOIN.
+ */
+ public boolean isLeftOuter() {
+ return leftOuter;
+ }
+
/** {@inheritDoc} */
@Override public String getSQL() {
StatementBuilder buff = new StatementBuilder();
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3ff16ab/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index e87a032..25d97fc 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -106,6 +106,9 @@ public class GridSqlQuerySplitter {
private Set<QueryTable> tbls = new HashSet<>();
/** */
+ private Set<String> pushedDownCols = new HashSet<>();
+
+ /** */
private boolean rdcQrySimple;
/** */
@@ -337,22 +340,75 @@ public class GridSqlQuerySplitter {
@SuppressWarnings("unused")
private static void debug(String label, String info) {
X.println();
- X.println(" ==" + label + "== ");
+ X.println(" == " + label + " == ");
X.println(info);
- X.println(" ======== ");
+ X.println(" ======================= ");
+ }
+
+ /**
+ * @param expr Expression.
+ * @return {@code true} If the expression contains pushed down columns.
+ */
+ private boolean hasPushedDownColumn(GridSqlAst expr) {
+ if (expr instanceof GridSqlColumn)
+ return pushedDownCols.contains(((GridSqlColumn)expr).columnName());
+
+ for (int i = 0; i < expr.size(); i++) {
+ if (hasPushedDownColumn(expr.child(i)))
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @param from FROM clause.
+ * @return {@code true} If contains LEFT OUTER JOIN.
+ */
+ private static boolean hasLeftJoin(GridSqlAst from) {
+ while (from instanceof GridSqlJoin) {
+ GridSqlJoin join = (GridSqlJoin)from;
+
+ assert !(join.rightTable() instanceof GridSqlJoin);
+
+ if (join.isLeftOuter())
+ return true;
+
+ from = join.leftTable();
+ }
+
+ return false;
}
/**
* @param qrym Query model for the SELECT.
*/
private void pushDownQueryModelSelect(QueryModel qrym) {
+ assert qrym.type == Type.SELECT: qrym.type;
+
+ boolean hasLeftJoin = hasLeftJoin(qrym.<GridSqlSelect>ast().from());
+
int begin = -1;
// Here we iterate over joined FROM table filters.
+ // !!! qrym.size() can change, never assign it to a variable.
for (int i = 0; i < qrym.size(); i++) {
QueryModel child = qrym.get(i);
- if (child.isQuery() && (child.needSplitChild || child.needSplit)) {
+ boolean hasPushedDownCol = false;
+
+ // It is either splittable subquery (it must remain in REDUCE query)
+ // or left join condition with pushed down columns (this condition
+ // can not be pushed down into a wrap query).
+ if ((child.isQuery() && (child.needSplitChild || child.needSplit)) ||
+ // We always must be at the right side of the join here to push down
+ // range on the left side. If there are no LEFT JOINs in the SELECT, then
+ // we will never have ON conditions, they are getting moved to WHERE clause.
+ (hasPushedDownCol = (hasLeftJoin && i != 0 && hasPushedDownColumn(findJoin(qrym, i).on())))) {
+ // Handle a single table push down case.
+ if (hasPushedDownCol && begin == -1)
+ begin = i - 1;
+
// Push down the currently collected range.
if (begin != -1) {
pushDownQueryModelRange(qrym, begin, i - 1);
@@ -361,8 +417,10 @@ public class GridSqlQuerySplitter {
assert qrym.get(i) == child; // Adjustment check: we have to return to the same point.
- // Reset range begin.
- begin = -1;
+ // Reset range begin: in case of pushed down column we can assume current child as
+ // as new begin (because it is not a splittable subquery), otherwise reset begin
+ // and try to find next range begin further.
+ begin = hasPushedDownCol ? i : -1;
}
if (child.needSplitChild)
@@ -394,7 +452,7 @@ public class GridSqlQuerySplitter {
for (int i = 0; i < qrym.size(); i++) {
QueryModel child = qrym.get(i);
- assert child.isQuery() : qrym.type;
+ assert child.isQuery() : child.type;
if (child.needSplit)
needSplitChild = true;
@@ -579,9 +637,12 @@ public class GridSqlQuerySplitter {
/**
* @param qrym Query model.
*/
- private void setNeedSplit(QueryModel qrym) {
- if (qrym.type == Type.SELECT)
+ private static void setNeedSplit(QueryModel qrym) {
+ if (qrym.type == Type.SELECT) {
+ assert !qrym.needSplitChild;
+
qrym.needSplit = true;
+ }
else if (qrym.type == Type.UNION) {
qrym.needSplitChild = true;
@@ -816,6 +877,8 @@ public class GridSqlQuerySplitter {
// Push down related ON conditions for all the related joins.
while (from instanceof GridSqlJoin) {
+ assert !(((GridSqlJoin)from).rightTable() instanceof GridSqlJoin);
+
pushDownColumnsInExpression(tblAliases, cols, wrapAlias, from, ON_CHILD);
from = from.child(LEFT_TABLE_CHILD);
@@ -909,6 +972,9 @@ public class GridSqlQuerySplitter {
// We have this map to avoid column duplicates in wrap query.
cols.put(uniqueColAlias, colAlias);
+
+ if (!pushedDownCols.add(uniqueColAlias)) // Must be globally unique.
+ throw new IllegalStateException(uniqueColAlias);
}
col = column(uniqueColAlias);
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3ff16ab/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index 8b743bf..6c61988 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -211,6 +211,56 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
/**
*/
+ public void testPushDownLeftJoin() {
+ IgniteCache<Integer, Person> c = ignite(0).getOrCreateCache(cacheConfig("ps", true,
+ Integer.class, Person.class));
+
+ try {
+ String subqryAgg = "(select max(p.id) as id, p.depId from Person p group by p.depId)";
+ String subqrySimple = "(select p.id, p.depId from Person p)";
+
+ for (int i = 0; i < 5; i++) {
+ for (int k = 0; k < 5; k++) {
+ SB qry = new SB("select * from ");
+
+ for (int j = 0; j < 5; j++) {
+ if (j != 0)
+ qry.a(j == i ? " left join " : " join ");
+
+ if (j == 2)
+ qry.a(subqryAgg);
+ else
+ qry.a(j == k ? subqrySimple : "Person");
+
+ qry.a(" p").a(j);
+
+ if (j != 0) {
+ qry.a(" on ");
+
+ qry.a(" p0.id").a(" = ").a("p").a(j).a(".depId");
+ }
+ }
+
+
+ X.println(" ---> ik: : " + i + " " + k);
+ X.println("\nqry: \n" + qry.toString());
+
+ c.query(new SqlFieldsQuery(qry.toString())
+ .setEnforceJoinOrder(true)).getAll();
+
+ X.println("\nPlan:\n" +
+ c.query(new SqlFieldsQuery("explain " + qry.toString())
+ .setEnforceJoinOrder(true)).getAll());
+ }
+ }
+ }
+ finally {
+ c.destroy();
+ }
+ }
+
+ /**
+ */
public void testReplicatedTablesUsingPartitionedCache() {
doTestReplicatedTablesUsingPartitionedCache(1, false, false);
}
[2/2] ignite git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/ignite
Posted by se...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/ignite
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4b0998e5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4b0998e5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4b0998e5
Branch: refs/heads/master
Commit: 4b0998e5d22a94f56db5ddc65ad7fc274a1e3199
Parents: e3ff16a a1e6edc
Author: Sergi Vladykin <se...@gmail.com>
Authored: Thu Jun 8 20:39:13 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Thu Jun 8 20:39:13 2017 +0300
----------------------------------------------------------------------
.../ClientTcpUnreachableMultiNodeSelfTest.java | 6 +-
.../configuration/CacheConfiguration.java | 24 +++
.../ignite/internal/binary/BinaryContext.java | 1 -
.../ignite/internal/binary/BinaryFieldEx.java | 5 +
.../ignite/internal/binary/BinaryFieldImpl.java | 5 +
.../ignite/internal/client/GridClientNode.java | 9 +-
.../client/impl/GridClientNodeImpl.java | 22 +-
.../connection/GridClientNioTcpConnection.java | 3 +-
.../PlatformDotNetConfigurationClosure.java | 8 +-
.../utils/PlatformConfigurationUtils.java | 8 +
.../query/DynamicTableAffinityKeyMapper.java | 92 ++++++++
.../processors/query/GridQueryProcessor.java | 59 +++---
.../internal/processors/query/QueryUtils.java | 19 +-
.../rest/client/message/GridClientNodeBean.java | 25 +++
.../top/GridTopologyCommandHandler.java | 1 +
.../query/h2/ddl/DdlStatementsProcessor.java | 5 +-
.../processors/query/h2/opt/GridH2Table.java | 5 +-
.../query/h2/sql/GridSqlCreateTable.java | 37 +++-
.../query/h2/sql/GridSqlQueryParser.java | 54 +++++
.../cache/index/H2DynamicTableSelfTest.java | 209 +++++++++++++++++--
.../Cache/CacheConfigurationTest.cs | 10 +
.../IgniteConfigurationSerializerTest.cs | 11 +-
.../IgniteConfigurationTest.cs | 4 +
.../Cache/Configuration/CacheConfiguration.cs | 23 ++
.../Datastream/IDataStreamer.cs | 8 +-
.../Apache.Ignite.Core/IgniteConfiguration.cs | 20 ++
.../IgniteConfigurationSection.xsd | 12 ++
modules/web-console/backend/.gitignore | 1 +
modules/web-console/frontend/.gitignore | 1 +
.../yardstick/cache/load/model/ModelUtil.java | 1 +
30 files changed, 621 insertions(+), 67 deletions(-)
----------------------------------------------------------------------