You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by ra...@apache.org on 2017/06/10 05:30:34 UTC
[4/5] lens git commit: LENS-1432 : Map Join tuning broken for this
query leading to Map timeouts
LENS-1432 : Map Join tuning broken for this query leading to Map timeouts
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/13ee2851
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/13ee2851
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/13ee2851
Branch: refs/heads/current-release-line
Commit: 13ee28519fbb8aa92c30abdff4a286155fddeef5
Parents: 419e190
Author: Sushil Mohanty <su...@gmail.com>
Authored: Tue Jun 6 18:14:39 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Sat Jun 10 13:30:08 2017 +0800
----------------------------------------------------------------------
.../cube/parse/MultiCandidateQueryWriterContext.java | 11 +++++++++++
.../org/apache/lens/cube/parse/QueryWriterContext.java | 4 ++++
.../java/org/apache/lens/driver/cube/RewriterPlan.java | 6 +++---
.../org/apache/lens/cube/parse/TestRewriterPlan.java | 2 ++
.../lens/cube/parse/TestUnionAndJoinCandidates.java | 2 +-
lens-cube/src/test/resources/schema/facts/b1fact1.xml | 2 +-
6 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
index d57c027..50046a5 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
@@ -18,8 +18,11 @@
*/
package org.apache.lens.cube.parse;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import org.apache.lens.cube.metadata.Dimension;
import org.apache.lens.server.api.error.LensException;
import com.google.common.collect.Lists;
@@ -90,4 +93,12 @@ public class MultiCandidateQueryWriterContext implements QueryWriterContext {
List<StorageCandidateHQLContext> leafWriterContexts = getLeafQueryWriterContexts();
return new UnionQueryWriter(leafWriterContexts, getCubeQueryContext());
}
+
+ public Map<Dimension, CandidateDim> getDimsToQuery() {
+ Map<Dimension, CandidateDim> allDimsQueried = new HashMap<>();
+ for (QueryWriterContext ctx : children) {
+ allDimsQueried.putAll(ctx.getDimsToQuery());
+ }
+ return allDimsQueried;
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
index d55de1f..6531f22 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
@@ -18,6 +18,9 @@
*/
package org.apache.lens.cube.parse;
+import java.util.Map;
+
+import org.apache.lens.cube.metadata.Dimension;
import org.apache.lens.server.api.error.LensException;
/**
@@ -31,4 +34,5 @@ public interface QueryWriterContext {
QueryAST getQueryAst();
void updateFromString() throws LensException;
QueryWriter toQueryWriter() throws LensException;
+ Map<Dimension, CandidateDim> getDimsToQuery();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
index caf8770..a27aec1 100644
--- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
+++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
@@ -45,10 +45,10 @@ public final class RewriterPlan extends DriverQueryPlan {
@SuppressWarnings("unchecked") // required for (Set<FactPartition>) casting
void extractPlan(Collection<CubeQueryContext> cubeQueries) {
-
for (CubeQueryContext ctx : cubeQueries) {
- if (ctx.getPickedDimTables() != null && !ctx.getPickedDimTables().isEmpty()) {
- for (CandidateDim dim : ctx.getPickedDimTables()) {
+ if (ctx.getQueryWriterContext().getDimsToQuery() != null
+ && !ctx.getQueryWriterContext().getDimsToQuery().isEmpty()) {
+ for (CandidateDim dim : ctx.getQueryWriterContext().getDimsToQuery().values()) {
addTablesQueried(dim.getStorageTable());
if (partitions.get(dim.getName()) == null || partitions.get(dim.getName()).isEmpty()) {
// puts storage table to latest part
http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java
index 76ea77d..6223df7 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java
@@ -71,6 +71,8 @@ public class TestRewriterPlan extends TestQueryRewrite {
+ " cubecity.name != \"XYZ\" and " + TWO_DAYS_RANGE + " having sum(msr2) > 1000 order by cubecity.name limit 50",
conf);
ctx.toHQL();
+ // One dimension table queried
+ Assert.assertEquals(ctx.getQueryWriterContext().getDimsToQuery().size(), 1);
RewriterPlan plan = new RewriterPlan(Collections.singleton(ctx));
Assert.assertNotNull(plan);
Assert.assertFalse(plan.getTablesQueried().isEmpty());
http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
index 429e1c6..dc06ead 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
@@ -147,7 +147,7 @@ public class TestUnionAndJoinCandidates extends TestQueryRewrite {
compareContains(outerGroupBy, rewrittenQuery);
}
- @Test(invocationCount = 100)
+ @Test
public void testFinalCandidateRewrittenQuery() throws ParseException, LensException {
try {
// Query with non projected measure in having clause.
http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/test/resources/schema/facts/b1fact1.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/b1fact1.xml b/lens-cube/src/test/resources/schema/facts/b1fact1.xml
index b51a34a..1500037 100644
--- a/lens-cube/src/test/resources/schema/facts/b1fact1.xml
+++ b/lens-cube/src/test/resources/schema/facts/b1fact1.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_fact_table name="b1fact1" cube_name="b1cube" weight="5.0" xmlns="uri:lens:cube:0.1">
+<x_fact_table name="b1fact1" cube_name="b1cube" weight="5.1" xmlns="uri:lens:cube:0.1">
<columns>
<column name="zipcode" _type="int" comment="zip"/>
<column name="cityid" _type="int" comment="city id"/>