You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by ra...@apache.org on 2017/06/10 05:30:31 UTC
[1/5] lens git commit: LENS-1427: ExpressionResolver is removing
expressions if they are not answerable by *any* candidates
Repository: lens
Updated Branches:
refs/heads/current-release-line be75e1d55 -> d3875b4e9
LENS-1427: ExpressionResolver is removing expressions if they are not answerable by *any* candidates
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/bd4b5529
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/bd4b5529
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/bd4b5529
Branch: refs/heads/current-release-line
Commit: bd4b5529ac15b51a6ef91f431b2037e63078ea34
Parents: be75e1d
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Tue Jun 6 15:37:41 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Sat Jun 10 13:28:54 2017 +0800
----------------------------------------------------------------------
.../lens/cube/parse/ExpressionResolver.java | 37 +++++++++-----------
.../test/resources/schema/cubes/base/b1cube.xml | 2 +-
.../test/resources/schema/cubes/base/b2cube.xml | 2 +-
.../resources/schema/cubes/base/testcube.xml | 2 +-
4 files changed, 19 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/bd4b5529/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index a2f4bb6..ea6d5c7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -550,15 +550,14 @@ class ExpressionResolver implements ContextRewriter {
}
// Remove expressions for which denormalized columns are no more reachable
esc.getDeNormCtx().pruneReferences(cubeql);
- for (String table : esc.getDeNormCtx().getTableToRefCols().keySet()) {
- Set<String> nonReachableFields = esc.getDeNormCtx().getNonReachableReferenceFields(table);
- if (!nonReachableFields.isEmpty()) {
- log.info("Removing expression {} as columns {} are not available", esc, nonReachableFields);
- iterator.remove();
- removedEsc.add(esc);
- removed = true;
- break;
- }
+ if (!esc.getDeNormCtx().getTableToRefCols().isEmpty()
+ && esc.getDeNormCtx().getTableToRefCols().keySet().stream()
+ .map(esc.getDeNormCtx()::getNonReachableReferenceFields).noneMatch(Set::isEmpty)) {
+ log.info("Removing expression {} as all tables have non reachable fields", esc);
+ iterator.remove();
+ removedEsc.add(esc);
+ removed = true;
+ break;
}
if (removed) {
continue;
@@ -662,19 +661,15 @@ class ExpressionResolver implements ContextRewriter {
for (Map.Entry<String, Set<ExpressionContext>> ecEntry : exprCtx.allExprsQueried.entrySet()) {
String expr = ecEntry.getKey();
Set<ExpressionContext> ecSet = ecEntry.getValue();
- for (ExpressionContext ec : ecSet) {
- if (ec.getSrcTable().getName().equals(cubeql.getCube().getName())) {
- for (Iterator<Candidate> sItr = cubeql.getCandidates().iterator(); sItr.hasNext();) {
- Candidate cand = sItr.next();
- if (!cand.isExpressionEvaluable(ec)) {
- log.info("Not considering Candidate :{} as {} is not evaluable", cand, ec.exprCol.getName());
- sItr.remove();
- cubeql.addCandidatePruningMsg(cand,
- CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName()));
- }
- }
+ cubeql.getCandidates().removeIf(x-> {
+ if (ecSet.stream().noneMatch(x::isExpressionEvaluable)) {
+ log.info("Not considering Candidate :{} as {} is not evaluable", x, expr);
+ cubeql.addCandidatePruningMsg(x,
+ CandidateTablePruneCause.expressionNotEvaluable(expr));
+ return true;
}
- }
+ return false;
+ });
}
}
// prune candidate dims without any valid expressions
http://git-wip-us.apache.org/repos/asf/lens/blob/bd4b5529/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml b/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
index e3e9909..fd25982 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
@@ -178,7 +178,7 @@
<expr_spec expr="cubecity.name"/>
</expression>
<expression _type="string" name="citysegmsr1" display_string="city segmsr1" description="city segmsr1 ">
- <expr_spec expr="sum(case when cubecity.name = 'foo' then segmsr1 end)"/>
+ <expr_spec expr="sum(case when singlecolchainfield = 'foo' then segmsr1 end)"/>
</expression>
<expression _type="double" name="msr8" display_string="Sixth Msr" description="measure expression">
<expr_spec expr="msr2 + msr3"/>
http://git-wip-us.apache.org/repos/asf/lens/blob/bd4b5529/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml b/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml
index e5cf2c0..ca6bcd7 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml
@@ -178,7 +178,7 @@
<expr_spec expr="cubecity.name"/>
</expression>
<expression _type="string" name="citysegmsr1" display_string="city segmsr1" description="city segmsr1 ">
- <expr_spec expr="sum(case when cubecity.name = 'foo' then segmsr1 end)"/>
+ <expr_spec expr="sum(case when singlecolchainfield = 'foo' then segmsr1 end)"/>
</expression>
<expression _type="double" name="msr8" display_string="Sixth Msr" description="measure expression">
<expr_spec expr="msr2 + msr3"/>
http://git-wip-us.apache.org/repos/asf/lens/blob/bd4b5529/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
index 54e39c2..088c8de 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
@@ -166,7 +166,7 @@
<expr_spec expr="cubecity.name"/>
</expression>
<expression _type="string" name="citysegmsr1" display_string="city segmsr1" description="city segmsr1 ">
- <expr_spec expr="sum(case when cubecity.name = 'foo' then segmsr1 end)"/>
+ <expr_spec expr="sum(case when singlecolchainfield = 'foo' then segmsr1 end)"/>
</expression>
<expression _type="double" name="msr8" display_string="Sixth Msr" description="measure expression">
<expr_spec expr="msr2 + msr3"/>
[3/5] lens git commit: LENS-1431 Data-availability feature fixes
Posted by ra...@apache.org.
LENS-1431 Data-availability feature fixes
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/419e1904
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/419e1904
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/419e1904
Branch: refs/heads/current-release-line
Commit: 419e1904ec971ff1c944f7543ea6bfc6607555e7
Parents: 60e1277
Author: Sushrut Meghshyam Ikhar <su...@gmail.com>
Authored: Tue Jun 6 17:53:27 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Sat Jun 10 13:30:01 2017 +0800
----------------------------------------------------------------------
.../lens/cube/parse/ExpressionResolver.java | 4 --
.../lens/cube/parse/StorageCandidate.java | 8 +++-
.../cube/parse/MockCompletenessChecker.java | 10 ++--
.../lens/cube/parse/TestCubeRewriter.java | 49 +++++++++++++++++++-
.../resources/schema/cubes/base/basecube.xml | 5 ++
.../resources/schema/cubes/derived/der1.xml | 1 +
.../schema/facts/testfact2_raw_base.xml | 1 +
.../schema/facts/virtual/virtualfact.xml | 27 +++++++++++
.../test/resources/schema/facts/virtualfact.xml | 27 -----------
9 files changed, 95 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index ea6d5c7..b1654d1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -556,12 +556,8 @@ class ExpressionResolver implements ContextRewriter {
log.info("Removing expression {} as all tables have non reachable fields", esc);
iterator.remove();
removedEsc.add(esc);
- removed = true;
break;
}
- if (removed) {
- continue;
- }
//remove expressions which are not valid in the timerange queried
// If an expression is defined as
// ex = a + b // from t1 to t2;
http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 3f5e2e5..7980797 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -651,6 +651,10 @@ public class StorageCandidate implements Candidate, CandidateTable {
}
private boolean evaluateMeasuresCompleteness(TimeRange timeRange) throws LensException {
+ if (getCubeMetastoreClient() == null || !getCubeMetastoreClient().isDataCompletenessCheckEnabled()) {
+ log.info("Skipping availability check for the fact table: {} as dataCompleteness check is not enabled", fact);
+ return true;
+ }
String factDataCompletenessTag = fact.getDataCompletenessTag();
if (factDataCompletenessTag == null) {
log.info("Not checking completeness for the fact table:{} as the dataCompletenessTag is not set", fact);
@@ -673,7 +677,8 @@ public class StorageCandidate implements Candidate, CandidateTable {
log.info("No Queried measures with the dataCompletenessTag, hence skipping the availability check");
return true;
}
- boolean isDataComplete = false;
+ // default completenessTag will be true
+ boolean isDataComplete = true;
DataCompletenessChecker completenessChecker = getCubeMetastoreClient().getCompletenessChecker();
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -695,6 +700,7 @@ public class StorageCandidate implements Candidate, CandidateTable {
String measureorExprFromTag = tagToMeasureOrExprMap.get(tag);
dataCompletenessMap.computeIfAbsent(measureorExprFromTag, k -> new HashMap<>())
.put(formatter.format(completenessResult.getKey()), completenessResult.getValue());
+ // set completeness to false if availability for measure is below threshold
isDataComplete = false;
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java
index 76e81d5..c1936eb 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java
@@ -33,10 +33,12 @@ public class MockCompletenessChecker implements DataCompletenessChecker {
Calendar cal = Calendar.getInstance();
cal.setTimeZone(TimeZone.getTimeZone("GMT"));
cal.add(Calendar.DATE, -1);
- if (factTag.equals("f1")) {
- partitionCompleteness.put(cal.getTime(), 80f);
- } else {
- partitionCompleteness.put(cal.getTime(), 90f);
+ if (start.before(cal.getTime()) && end.after(cal.getTime())) {
+ if (factTag.equals("f1")) {
+ partitionCompleteness.put(cal.getTime(), 80f);
+ } else {
+ partitionCompleteness.put(cal.getTime(), 90f);
+ }
}
result.put("tag1", partitionCompleteness);
return result;
http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 65ca861..d8c7335 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -981,7 +981,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
conf.setStrings(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL, "dt");
String hqlQuery = rewrite("select SUM(msr9) from basecube where " + TWO_DAYS_RANGE, conf);
String expected = getExpectedQuery("basecube", "select sum(basecube.msr9) as `sum(msr9)` FROM ", null, null,
- getWhereForHourly2days("basecube", "c1_testfact5_raw_base"));
+ getWhereForHourly2days("basecube", "c1_testfact5_raw_base"));
compareQueries(hqlQuery, expected);
}
@@ -1007,6 +1007,53 @@ public class TestCubeRewriter extends TestQueryRewrite {
}
}
+ /*
+ * The test is to check that query is rewritten successfully if there is missing entry in
+ * dataavailability service for the measure's tag
+ */
+ @Test
+ public void testQueryWithMeasureWithDataCompletenessTagWithDataAvailiability() throws ParseException,
+ LensException {
+ NoCandidateFactAvailableException ne;
+ PruneCauses.BriefAndDetailedError pruneCauses;
+ String hqlQuery;
+ String expected;
+
+ Configuration conf = getConf();
+ conf.setStrings(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL, "dt");
+
+ conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true);
+
+ // 1. data completeness check applicable
+ ne = getLensExceptionInRewrite("select SUM(msr16) from basecube" + " where " + TWO_DAYS_RANGE, conf);
+ pruneCauses = ne.getJsonMessage();
+ assertEquals(pruneCauses.getBrief().substring(0, 10), INCOMPLETE_PARTITION.errorFormat.substring(0, 10),
+ pruneCauses.getBrief());
+
+ // 2. time outside data completeness check but missing partitions
+ ne = getLensExceptionInRewrite("select SUM(msr16) from basecube where " + TWO_DAYS_RANGE_BEFORE_4_DAYS, conf);
+ pruneCauses = ne.getJsonMessage();
+ assertEquals(pruneCauses.getBrief().substring(0, 10), MISSING_PARTITIONS.errorFormat.substring(0, 10),
+ pruneCauses.getBrief());
+
+
+ conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false);
+
+ // 3. query allowed on partial data although data incomplete
+ hqlQuery = rewrite("select SUM(msr16) from basecube" + " where " + TWO_DAYS_RANGE, conf);
+ expected = getExpectedQuery("basecube", "select sum(basecube.msr16) as `sum(msr16)` FROM ", null, null,
+ getWhereForHourly2days("basecube", "c1_testfact2_raw_base"));
+ compareQueries(hqlQuery.toLowerCase(), expected.toLowerCase());
+
+ // 4. query allowed on partial data with missing partitions but outside data availability window
+ hqlQuery = rewrite("select SUM(msr16) from basecube" + " where " + TWO_DAYS_RANGE_BEFORE_4_DAYS, conf);
+ expected = getExpectedQuery("basecube", "select sum(basecube.msr16) as `sum(msr16)` FROM ", null, null,
+ getWhereForUpdatePeriods("basecube", "c1_testfact2_raw_base",
+ DateUtils.addHours(getDateWithOffset(UpdatePeriod.DAILY, -6), -1),
+ getDateWithOffset(UpdatePeriod.DAILY, -4), Sets.newHashSet(UpdatePeriod.HOURLY)));
+ compareQueries(hqlQuery.toLowerCase(), expected.toLowerCase());
+ }
+
@Test
public void testCubeWhereQueryForMonthWithNoPartialData() throws Exception {
Configuration conf = getConf();
http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
index 708b510..bcea938 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
@@ -90,6 +90,11 @@
</measure>
<measure _type="INT" name="segmsr1" default_aggr="SUM" description="segmentation measure 1"
display_string="segmentation measure 1"/>
+ <measure _type="BIGINT" name="msr16" description="sixteenth measure">
+ <tags>
+ <property name="cube.measure.datacompleteness.tag" value="tag1"/>
+ </tags>
+ </measure>
</measures>
<dim_attributes>
<dim_attribute _type="string" name="business" display_string="business unit"
http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/resources/schema/cubes/derived/der1.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/derived/der1.xml b/lens-cube/src/test/resources/schema/cubes/derived/der1.xml
index cf36cc7..01bec03 100644
--- a/lens-cube/src/test/resources/schema/cubes/derived/der1.xml
+++ b/lens-cube/src/test/resources/schema/cubes/derived/der1.xml
@@ -33,6 +33,7 @@
<measure_name>msr1</measure_name>
<measure_name>msr11</measure_name>
<measure_name>msr9</measure_name>
+ <measure_name>msr16</measure_name>
</measure_names>
<dim_attr_names>
<attr_name>dim1</attr_name>
http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml b/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml
index 0bab461..69fd8d9 100644
--- a/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml
+++ b/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml
@@ -24,6 +24,7 @@
<column name="msr11" _type="int" comment="first measure"/>
<column name="msr12" _type="float" comment="second measure"/>
<column name="msr9" _type="bigint" comment="ninth measure"/>
+ <column name="msr16" _type="bigint" comment="sixteenth measure"/>
<column name="d_time" _type="timestamp" comment="event time"/>
<column name="processing_time" _type="timestamp" comment="processing time"/>
<column name="dim1" _type="string" comment="base dim"/>
http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml b/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
new file mode 100644
index 0000000..860965f
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_virtual_fact_table source_fact_name="summary1" cube_name="virtualcube" name="virtualfact1" xmlns="uri:lens:cube:0.1"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <properties>
+ <property name="cube.fact.query.where.filter" value=" dim1 = 10 "/>
+ </properties>
+</x_virtual_fact_table>
http://git-wip-us.apache.org/repos/asf/lens/blob/419e1904/lens-cube/src/test/resources/schema/facts/virtualfact.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/virtualfact.xml b/lens-cube/src/test/resources/schema/facts/virtualfact.xml
deleted file mode 100644
index 860965f..0000000
--- a/lens-cube/src/test/resources/schema/facts/virtualfact.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<x_virtual_fact_table source_fact_name="summary1" cube_name="virtualcube" name="virtualfact1" xmlns="uri:lens:cube:0.1"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
- <properties>
- <property name="cube.fact.query.where.filter" value=" dim1 = 10 "/>
- </properties>
-</x_virtual_fact_table>
[4/5] lens git commit: LENS-1432 : Map Join tuning broken for this
query leading to Map timeouts
Posted by ra...@apache.org.
LENS-1432 : Map Join tuning broken for this query leading to Map timeouts
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/13ee2851
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/13ee2851
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/13ee2851
Branch: refs/heads/current-release-line
Commit: 13ee28519fbb8aa92c30abdff4a286155fddeef5
Parents: 419e190
Author: Sushil Mohanty <su...@gmail.com>
Authored: Tue Jun 6 18:14:39 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Sat Jun 10 13:30:08 2017 +0800
----------------------------------------------------------------------
.../cube/parse/MultiCandidateQueryWriterContext.java | 11 +++++++++++
.../org/apache/lens/cube/parse/QueryWriterContext.java | 4 ++++
.../java/org/apache/lens/driver/cube/RewriterPlan.java | 6 +++---
.../org/apache/lens/cube/parse/TestRewriterPlan.java | 2 ++
.../lens/cube/parse/TestUnionAndJoinCandidates.java | 2 +-
lens-cube/src/test/resources/schema/facts/b1fact1.xml | 2 +-
6 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
index d57c027..50046a5 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
@@ -18,8 +18,11 @@
*/
package org.apache.lens.cube.parse;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import org.apache.lens.cube.metadata.Dimension;
import org.apache.lens.server.api.error.LensException;
import com.google.common.collect.Lists;
@@ -90,4 +93,12 @@ public class MultiCandidateQueryWriterContext implements QueryWriterContext {
List<StorageCandidateHQLContext> leafWriterContexts = getLeafQueryWriterContexts();
return new UnionQueryWriter(leafWriterContexts, getCubeQueryContext());
}
+
+ public Map<Dimension, CandidateDim> getDimsToQuery() {
+ Map<Dimension, CandidateDim> allDimsQueried = new HashMap<>();
+ for (QueryWriterContext ctx : children) {
+ allDimsQueried.putAll(ctx.getDimsToQuery());
+ }
+ return allDimsQueried;
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
index d55de1f..6531f22 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
@@ -18,6 +18,9 @@
*/
package org.apache.lens.cube.parse;
+import java.util.Map;
+
+import org.apache.lens.cube.metadata.Dimension;
import org.apache.lens.server.api.error.LensException;
/**
@@ -31,4 +34,5 @@ public interface QueryWriterContext {
QueryAST getQueryAst();
void updateFromString() throws LensException;
QueryWriter toQueryWriter() throws LensException;
+ Map<Dimension, CandidateDim> getDimsToQuery();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
index caf8770..a27aec1 100644
--- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
+++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
@@ -45,10 +45,10 @@ public final class RewriterPlan extends DriverQueryPlan {
@SuppressWarnings("unchecked") // required for (Set<FactPartition>) casting
void extractPlan(Collection<CubeQueryContext> cubeQueries) {
-
for (CubeQueryContext ctx : cubeQueries) {
- if (ctx.getPickedDimTables() != null && !ctx.getPickedDimTables().isEmpty()) {
- for (CandidateDim dim : ctx.getPickedDimTables()) {
+ if (ctx.getQueryWriterContext().getDimsToQuery() != null
+ && !ctx.getQueryWriterContext().getDimsToQuery().isEmpty()) {
+ for (CandidateDim dim : ctx.getQueryWriterContext().getDimsToQuery().values()) {
addTablesQueried(dim.getStorageTable());
if (partitions.get(dim.getName()) == null || partitions.get(dim.getName()).isEmpty()) {
// puts storage table to latest part
http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java
index 76ea77d..6223df7 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java
@@ -71,6 +71,8 @@ public class TestRewriterPlan extends TestQueryRewrite {
+ " cubecity.name != \"XYZ\" and " + TWO_DAYS_RANGE + " having sum(msr2) > 1000 order by cubecity.name limit 50",
conf);
ctx.toHQL();
+ // One dimension table queried
+ Assert.assertEquals(ctx.getQueryWriterContext().getDimsToQuery().size(), 1);
RewriterPlan plan = new RewriterPlan(Collections.singleton(ctx));
Assert.assertNotNull(plan);
Assert.assertFalse(plan.getTablesQueried().isEmpty());
http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
index 429e1c6..dc06ead 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
@@ -147,7 +147,7 @@ public class TestUnionAndJoinCandidates extends TestQueryRewrite {
compareContains(outerGroupBy, rewrittenQuery);
}
- @Test(invocationCount = 100)
+ @Test
public void testFinalCandidateRewrittenQuery() throws ParseException, LensException {
try {
// Query with non projected measure in having clause.
http://git-wip-us.apache.org/repos/asf/lens/blob/13ee2851/lens-cube/src/test/resources/schema/facts/b1fact1.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/b1fact1.xml b/lens-cube/src/test/resources/schema/facts/b1fact1.xml
index b51a34a..1500037 100644
--- a/lens-cube/src/test/resources/schema/facts/b1fact1.xml
+++ b/lens-cube/src/test/resources/schema/facts/b1fact1.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_fact_table name="b1fact1" cube_name="b1cube" weight="5.0" xmlns="uri:lens:cube:0.1">
+<x_fact_table name="b1fact1" cube_name="b1cube" weight="5.1" xmlns="uri:lens:cube:0.1">
<columns>
<column name="zipcode" _type="int" comment="zip"/>
<column name="cityid" _type="int" comment="city id"/>
[2/5] lens git commit: LENS-1435: Cube xml files are missing in
lens-example
Posted by ra...@apache.org.
LENS-1435: Cube xml files are missing in lens-example
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/60e1277f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/60e1277f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/60e1277f
Branch: refs/heads/current-release-line
Commit: 60e1277f7b7d5cb3ff5083f3fdf34dfae44dd0cb
Parents: bd4b552
Author: Sushil Mohanty <su...@gmail.com>
Authored: Tue Jun 6 17:45:22 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Sat Jun 10 13:29:20 2017 +0800
----------------------------------------------------------------------
lens-examples/src/main/resources/cube11.xml | 53 ++++++++++++++++++++++++
lens-examples/src/main/resources/cube22.xml | 53 ++++++++++++++++++++++++
lens-examples/src/main/resources/cube33.xml | 53 ++++++++++++++++++++++++
3 files changed, 159 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/60e1277f/lens-examples/src/main/resources/cube11.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/cube11.xml b/lens-examples/src/main/resources/cube11.xml
new file mode 100644
index 0000000..2283939
--- /dev/null
+++ b/lens-examples/src/main/resources/cube11.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_base_cube name="cube11" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <properties>
+ <property name="cube.sample_cube.timed.dimensions.list" value="dt"/>
+ </properties>
+ <measures>
+ <measure name="measure1" _type="BIGINT"/>
+ </measures>
+ <dim_attributes>
+ <dim_attribute name="dim1" _type="INT"/>
+ </dim_attributes>
+ <expressions>
+ <expression name="expr_msr5" _type="DOUBLE">
+ <expr_spec expr = "measure3 + measure4" start_time='2013-12-12T00:00:00' />
+ <expr_spec expr = "measure3 + measure4 + 0.01" end_time='2013-12-12T00:00:00'/>
+ </expression>
+ </expressions>
+ <join_chains>
+ <join_chain name="sample_dim_chain">
+ <paths>
+ <path>
+ <edges>
+ <edge>
+ <from table="sample_cube" column="dim3" />
+ <to table="sample_dim" column="id" />
+ </edge>
+ </edges>
+ </path>
+ </paths>
+ </join_chain>
+ </join_chains>
+</x_base_cube>
http://git-wip-us.apache.org/repos/asf/lens/blob/60e1277f/lens-examples/src/main/resources/cube22.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/cube22.xml b/lens-examples/src/main/resources/cube22.xml
new file mode 100644
index 0000000..59b1828
--- /dev/null
+++ b/lens-examples/src/main/resources/cube22.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_base_cube name="cube22" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <properties>
+ <property name="cube.sample_cube.timed.dimensions.list" value="dt"/>
+ </properties>
+ <measures>
+ <measure name="measure1" _type="BIGINT"/>
+ </measures>
+ <dim_attributes>
+ <dim_attribute name="dim1" _type="INT"/>
+ </dim_attributes>
+ <expressions>
+ <expression name="expr_msr5" _type="DOUBLE">
+ <expr_spec expr = "measure3 + measure4" start_time='2013-12-12T00:00:00' />
+ <expr_spec expr = "measure3 + measure4 + 0.01" end_time='2013-12-12T00:00:00'/>
+ </expression>
+ </expressions>
+ <join_chains>
+ <join_chain name="sample_dim_chain">
+ <paths>
+ <path>
+ <edges>
+ <edge>
+ <from table="sample_cube" column="dim3" />
+ <to table="sample_dim" column="id" />
+ </edge>
+ </edges>
+ </path>
+ </paths>
+ </join_chain>
+ </join_chains>
+</x_base_cube>
http://git-wip-us.apache.org/repos/asf/lens/blob/60e1277f/lens-examples/src/main/resources/cube33.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/cube33.xml b/lens-examples/src/main/resources/cube33.xml
new file mode 100644
index 0000000..47b9b17
--- /dev/null
+++ b/lens-examples/src/main/resources/cube33.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_base_cube name="cube33" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <properties>
+ <property name="cube.sample_cube.timed.dimensions.list" value="dt"/>
+ </properties>
+ <measures>
+ <measure name="measure1" _type="BIGINT"/>
+ </measures>
+ <dim_attributes>
+ <dim_attribute name="dim1" _type="INT"/>
+ </dim_attributes>
+ <expressions>
+ <expression name="expr_msr5" _type="DOUBLE">
+ <expr_spec expr = "measure3 + measure4" start_time='2013-12-12T00:00:00' />
+ <expr_spec expr = "measure3 + measure4 + 0.01" end_time='2013-12-12T00:00:00'/>
+ </expression>
+ </expressions>
+ <join_chains>
+ <join_chain name="sample_dim_chain">
+ <paths>
+ <path>
+ <edges>
+ <edge>
+ <from table="sample_cube" column="dim3" />
+ <to table="sample_dim" column="id" />
+ </edge>
+ </edges>
+ </path>
+ </paths>
+ </join_chain>
+ </join_chains>
+</x_base_cube>
[5/5] lens git commit: LENS-1434: Segmentation Candidate should have
dynamic cost depending on facts picked in segmented cubes
Posted by ra...@apache.org.
LENS-1434: Segmentation Candidate should have dynamic cost depending on facts picked in segmented cubes
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d3875b4e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d3875b4e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d3875b4e
Branch: refs/heads/current-release-line
Commit: d3875b4e9f46b97114819ea3318d76c1b25cc62b
Parents: 13ee285
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Tue Jun 6 18:25:05 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Sat Jun 10 13:30:24 2017 +0800
----------------------------------------------------------------------
.../org/apache/lens/cube/parse/Candidate.java | 2 +-
.../lens/cube/parse/CubeQueryRewriter.java | 12 +++----
.../apache/lens/cube/parse/JoinCandidate.java | 12 +++++--
.../lens/cube/parse/LightestFactResolver.java | 36 +++++++++++---------
.../lens/cube/parse/SegmentationCandidate.java | 19 +++++++++--
.../lens/cube/parse/StorageCandidate.java | 22 +++++++-----
.../cube/parse/StorageCandidateHQLContext.java | 9 +++++
.../apache/lens/cube/parse/UnionCandidate.java | 15 +++++---
.../lens/cube/parse/UnionQueryWriter.java | 17 +++++----
9 files changed, 95 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/d3875b4e/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
index 0855ced..ffa8fb6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
@@ -105,7 +105,7 @@ public interface Candidate {
/**
* @return the cost of this candidate
*/
- double getCost();
+ OptionalDouble getCost();
/**
* Returns true if this candidate contains the given candidate
http://git-wip-us.apache.org/repos/asf/lens/blob/d3875b4e/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
index d064cdb..0ef41f3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
@@ -149,6 +149,7 @@ public class CubeQueryRewriter {
DenormalizationResolver denormResolver = new DenormalizationResolver();
CandidateTableResolver candidateTblResolver = new CandidateTableResolver();
StorageTableResolver storageTableResolver = new StorageTableResolver(conf);
+ LightestFactResolver lightestFactResolver = new LightestFactResolver();
// Phase 1 of exprResolver: Resolve expressions
rewriters.add(exprResolver);
@@ -185,7 +186,7 @@ public class CubeQueryRewriter {
rewriters.add(exprResolver);
// Pick the least cost combination(s) (and prune others) out of a set of combinations produced
// by CandidateCoveringSetsResolver
- rewriters.add(new LightestFactResolver());
+ rewriters.add(lightestFactResolver);
}
// Phase 2 of storageTableResolver: resolve storage table partitions.
@@ -206,11 +207,10 @@ public class CubeQueryRewriter {
// Phase 2 of exprResolver : Prune candidate facts without any valid expressions
rewriters.add(exprResolver);
- if (!lightFactFirst) {
- // Pick the least cost combination(s) (and prune others) out of a set of combinations produced
- // by CandidateCoveringSetsResolver
- rewriters.add(new LightestFactResolver());
- }
+ // Pick the least cost combination(s) (and prune others) out of a set of combinations produced
+ // by CandidateCoveringSetsResolver
+ rewriters.add(lightestFactResolver);
+
// if two combinations have the same least weight/cost, then the combination with least number of time partitions
// queried will be picked. Rest of the combinations will be pruned
rewriters.add(new LeastPartitionResolver());
http://git-wip-us.apache.org/repos/asf/lens/blob/d3875b4e/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
index c4049cd..d9915f4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
@@ -72,8 +72,16 @@ public class JoinCandidate implements Candidate {
}
@Override
- public double getCost() {
- return children.stream().mapToDouble(Candidate::getCost).sum();
+ public OptionalDouble getCost() {
+ double cost = 0;
+ for (Candidate candidate : getChildren()) {
+ if (candidate.getCost().isPresent()) {
+ cost += candidate.getCost().getAsDouble();
+ } else {
+ return OptionalDouble.empty();
+ }
+ }
+ return OptionalDouble.of(cost);
}
@Override
http://git-wip-us.apache.org/repos/asf/lens/blob/d3875b4e/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java
index dd25f3e..52e3632 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java
@@ -19,7 +19,11 @@
package org.apache.lens.cube.parse;
-import java.util.*;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
import org.apache.lens.server.api.error.LensException;
@@ -35,21 +39,21 @@ public class LightestFactResolver implements ContextRewriter {
@Override
public void rewriteContext(CubeQueryContext cubeql) throws LensException {
if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) {
- Map<Candidate, Double> factWeightMap = new HashMap<Candidate, Double>();
-
- for (Candidate cand : cubeql.getCandidates()) {
- factWeightMap.put(cand, cand.getCost());
- }
-
- double minWeight = Collections.min(factWeightMap.values());
-
- for (Iterator<Candidate> i = cubeql.getCandidates().iterator(); i.hasNext();) {
- Candidate cand = i.next();
- if (factWeightMap.get(cand) > minWeight) {
- log.info("Not considering candidate:{} from final candidates as it has more fact weight:{} minimum:{}",
- cand, factWeightMap.get(cand), minWeight);
- cubeql.addCandidatePruningMsg(cand, new CandidateTablePruneCause(CandidateTablePruneCode.MORE_WEIGHT));
- i.remove();
+ Map<Candidate, Double> factWeightMap = cubeql.getCandidates().stream()
+ .filter(candidate -> candidate.getCost().isPresent())
+ .collect(Collectors.toMap(Function.identity(), x -> x.getCost().getAsDouble()));
+ if (!factWeightMap.isEmpty()) {
+ double minWeight = Collections.min(factWeightMap.values());
+ for (Iterator<Candidate> i = cubeql.getCandidates().iterator(); i.hasNext();) {
+ Candidate cand = i.next();
+ if (factWeightMap.containsKey(cand)) {
+ if (factWeightMap.get(cand) > minWeight) {
+ log.info("Not considering candidate:{} from final candidates as it has more fact weight:{} minimum:{}",
+ cand, factWeightMap.get(cand), minWeight);
+ cubeql.addCandidatePruningMsg(cand, new CandidateTablePruneCause(CandidateTablePruneCode.MORE_WEIGHT));
+ i.remove();
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/d3875b4e/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
index a359d86..a2bd485 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
@@ -37,6 +37,7 @@ import java.util.Date;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.OptionalDouble;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collector;
@@ -238,8 +239,20 @@ public class SegmentationCandidate implements Candidate {
}
@Override
- public double getCost() {
- return segmentation.weight();
+ public OptionalDouble getCost() {
+ if (areCandidatesPicked()) {
+ double cost = 0.0;
+ for (Candidate candidate : getChildren()) {
+ if (candidate.getCost().isPresent()) {
+ cost += candidate.getCost().getAsDouble();
+ } else {
+ return OptionalDouble.empty();
+ }
+ }
+ return OptionalDouble.of(cost);
+ } else {
+ return OptionalDouble.empty();
+ }
}
@Override
@@ -281,7 +294,7 @@ public class SegmentationCandidate implements Candidate {
// I can't ask my children to check this context for evaluability.
return cubeStream()
.map(cube -> cube.getExpressionByName(expr.getExprCol().getName()))
- .allMatch(Predicate.isEqual(expr.getExprCol()));
+ .allMatch(Objects::nonNull);
}
private boolean areCandidatesPicked() {
http://git-wip-us.apache.org/repos/asf/lens/blob/d3875b4e/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 7980797..c8ff3b8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -41,6 +41,7 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.OptionalDouble;
import java.util.Set;
import java.util.TimeZone;
import java.util.TreeSet;
@@ -298,6 +299,7 @@ public class StorageCandidate implements Candidate, CandidateTable {
return (AbstractCubeTable) cube;
}
+ @Override
public StorageCandidate copy() throws LensException {
return new StorageCandidate(this);
}
@@ -307,6 +309,7 @@ public class StorageCandidate implements Candidate, CandidateTable {
return phrase.isEvaluable(this);
}
+ @Override
public AbstractCubeTable getTable() {
return (AbstractCubeTable) fact;
}
@@ -350,8 +353,8 @@ public class StorageCandidate implements Candidate, CandidateTable {
}
@Override
- public double getCost() {
- return fact.weight();
+ public OptionalDouble getCost() {
+ return OptionalDouble.of(fact.weight());
}
@Override
@@ -367,7 +370,7 @@ public class StorageCandidate implements Candidate, CandidateTable {
private void updatePartitionStorage(FactPartition part) throws LensException {
try {
if (getCubeMetastoreClient().factPartitionExists(fact, part, storageTable)) {
- part.getStorageTables().add(name);
+ part.getStorageTables().add(storageTable);
part.setFound(true);
}
} catch (HiveException e) {
@@ -417,10 +420,10 @@ public class StorageCandidate implements Candidate, CandidateTable {
&& cubeQueryContext.getRangeWriter().getClass().equals(BetweenTimeRangeWriter.class)) {
FactPartition part = new FactPartition(partCol, fromDate, maxInterval, null, partWhereClauseFormat);
partitions.add(part);
- part.getStorageTables().add(storageName);
+ part.getStorageTables().add(storageTable);
part = new FactPartition(partCol, toDate, maxInterval, null, partWhereClauseFormat);
partitions.add(part);
- part.getStorageTables().add(storageName);
+ part.getStorageTables().add(storageTable);
this.participatingUpdatePeriods.add(maxInterval);
log.info("Added continuous fact partition for storage table {}", storageName);
return true;
@@ -534,7 +537,7 @@ public class StorageCandidate implements Candidate, CandidateTable {
missingPartitions.add(part);
if (!failOnPartialData) {
partitions.add(part);
- part.getStorageTables().add(storageName);
+ part.getStorageTables().add(storageTable);
}
} else {
log.info("No finer granualar partitions exist for {}", part);
@@ -735,7 +738,6 @@ public class StorageCandidate implements Candidate, CandidateTable {
@Override
public boolean isDimAttributeEvaluable(String dim) throws LensException {
-
return getCubeQueryContext().getDeNormCtx()
.addRefUsage(getCubeQueryContext(), this, dim, getCubeQueryContext().getCube().getName());
}
@@ -929,9 +931,9 @@ public class StorageCandidate implements Candidate, CandidateTable {
StorageCandidate updatePeriodSpecificSc;
for (UpdatePeriod period : participatingUpdatePeriods) {
updatePeriodSpecificSc = copy();
- updatePeriodSpecificSc.truncatePartitions(period);
updatePeriodSpecificSc.setResolvedName(getCubeMetastoreClient().getStorageTableName(fact.getName(),
storageName, period));
+ updatePeriodSpecificSc.truncatePartitions(period);
periodSpecificScList.add(updatePeriodSpecificSc);
}
periodSpecificStorageCandidates = periodSpecificScList;
@@ -949,6 +951,10 @@ public class StorageCandidate implements Candidate, CandidateTable {
while (rangeItr.hasNext()) {
Map.Entry<TimeRange, Set<FactPartition>> rangeEntry = rangeItr.next();
rangeEntry.getValue().removeIf(factPartition -> !factPartition.getPeriod().equals(updatePeriod));
+ rangeEntry.getValue().forEach(factPartition -> {
+ factPartition.getStorageTables().remove(storageTable);
+ factPartition.getStorageTables().add(resolvedName);
+ });
if (rangeEntry.getValue().isEmpty()) {
rangeItr.remove();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/d3875b4e/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
index c535196..cca39c0 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
@@ -150,4 +150,13 @@ public class StorageCandidateHQLContext extends DimHQLContext {
}
}
}
+
+ @Override
+ public int hashCode() {
+ final int PRIME = 59;
+ int result = 1;
+ result = result * PRIME + getStorageCandidate().hashCode();
+ result = result * PRIME + getCube().hashCode();
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/d3875b4e/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java
index 757a877..510ea0c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java
@@ -30,6 +30,7 @@ import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Optional;
+import java.util.OptionalDouble;
import java.util.Set;
import org.apache.lens.cube.metadata.FactPartition;
@@ -154,14 +155,18 @@ public class UnionCandidate implements Candidate {
}
@Override
- public double getCost() {
+ public OptionalDouble getCost() {
double cost = 0.0;
for (TimeRange timeRange : getCubeQueryContext().getTimeRanges()) {
for (Map.Entry<Candidate, TimeRange> entry : getTimeRangeSplit(timeRange).entrySet()) {
- cost += entry.getKey().getCost() * entry.getValue().milliseconds() / timeRange.milliseconds();
+ if (entry.getKey().getCost().isPresent()) {
+ cost += entry.getKey().getCost().getAsDouble() *entry.getValue().milliseconds() / timeRange.milliseconds();
+ } else {
+ return OptionalDouble.empty();
+ }
}
}
- return cost;
+ return OptionalDouble.of(cost);
}
@Override
@@ -256,7 +261,9 @@ public class UnionCandidate implements Candidate {
* @return
*/
private Map<Candidate, TimeRange> splitTimeRangeForChildren(TimeRange timeRange) {
- children.sort(comparing(Candidate::getCost));
+ if (children.stream().map(Candidate::getCost).allMatch(OptionalDouble::isPresent)) {
+ children.sort(comparing(x -> x.getCost().getAsDouble()));
+ }
Map<Candidate, TimeRange> childrenTimeRangeMap = new HashMap<>();
// Sorted list based on the weights.
Set<TimeRange> ranges = new HashSet<>();
http://git-wip-us.apache.org/repos/asf/lens/blob/d3875b4e/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index 9412f27..cc0a2e5 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -48,7 +48,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
private Map<HQLParser.HashableASTNode, ASTNode> innerToOuterSelectASTs = new HashMap<>();
private Map<HQLParser.HashableASTNode, ASTNode> innerToOuterHavingASTs = new HashMap<>();
- private Map<String, ASTNode> storageCandidateToSelectAstMap = new HashMap<>();
+ private Map<StorageCandidateHQLContext, ASTNode> storageCandidateToSelectAstMap = new HashMap<>();
private CubeQueryContext cubeql;
static final ASTNode DEFAULT_MEASURE_AST;
private static final String DEFAULT_MEASURE = "0.0";
@@ -91,8 +91,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
*/
private void updateAsts() {
for (StorageCandidateHQLContext sc : storageCandidates) {
- storageCandidateToSelectAstMap.put(sc.getStorageCandidate().toString(),
- new ASTNode(new CommonToken(TOK_SELECT, "TOK_SELECT")));
+ storageCandidateToSelectAstMap.put(sc, new ASTNode(new CommonToken(TOK_SELECT, "TOK_SELECT")));
if (sc.getQueryAst().getHavingAST() != null) {
cubeql.setHavingAST(sc.getQueryAst().getHavingAST());
}
@@ -403,7 +402,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
if (!phrase.hasMeasures(cubeql)) {
for (StorageCandidateHQLContext sc : storageCandidates) {
ASTNode exprWithOutAlias = (ASTNode) sc.getQueryAst().getSelectAST().getChild(i).getChild(0);
- storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()).
+ storageCandidateToSelectAstMap.get(sc).
addChild(getSelectExpr(exprWithOutAlias, aliasNode, false));
}
@@ -412,7 +411,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
for (StorageCandidateHQLContext sc : storageCandidates) {
if (sc.getStorageCandidate().getAnswerableMeasurePhraseIndices().contains(phrase.getPosition())) {
ASTNode exprWithOutAlias = (ASTNode) sc.getQueryAst().getSelectAST().getChild(i).getChild(0);
- storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()).
+ storageCandidateToSelectAstMap.get(sc).
addChild(getSelectExpr(exprWithOutAlias, aliasNode, false));
} else {
ASTNode resolvedExprNode = getAggregateNodesExpression(i);
@@ -421,7 +420,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
} else {
resolvedExprNode = getSelectExpr(null, null, true);
}
- storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()).
+ storageCandidateToSelectAstMap.get(sc).
addChild(getSelectExpr(resolvedExprNode, aliasNode, false));
}
}
@@ -431,7 +430,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
for (StorageCandidateHQLContext sc : storageCandidates) {
if (sc.getStorageCandidate().getAnswerableMeasurePhraseIndices().contains(phrase.getPosition())) {
ASTNode exprWithOutAlias = (ASTNode) sc.getQueryAst().getSelectAST().getChild(i).getChild(0);
- storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()).
+ storageCandidateToSelectAstMap.get(sc).
addChild(getSelectExpr(exprWithOutAlias, aliasNode, false));
} else {
ASTNode resolvedExprNode = getAggregateNodesExpression(i);
@@ -440,7 +439,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
} else {
resolvedExprNode = getSelectExpr(null, null, true);
}
- storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString()).
+ storageCandidateToSelectAstMap.get(sc).
addChild(getSelectExpr(resolvedExprNode, aliasNode, false));
}
}
@@ -485,7 +484,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
private void processSelectExpression(StorageCandidateHQLContext sc, ASTNode outerSelectAst, ASTNode innerSelectAST,
AliasDecider aliasDecider) throws LensException {
//ASTNode selectAST = sc.getQueryAst().getSelectAST();
- ASTNode selectAST = storageCandidateToSelectAstMap.get(sc.getStorageCandidate().toString());
+ ASTNode selectAST = storageCandidateToSelectAstMap.get(sc);
if (selectAST == null) {
return;
}