You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2017/06/06 12:24:10 UTC
lens git commit: LENS-1431 Data-availability feature fixes
Repository: lens
Updated Branches:
refs/heads/master aaf455111 -> 359937fab
LENS-1431 Data-availability feature fixes
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/359937fa
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/359937fa
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/359937fa
Branch: refs/heads/master
Commit: 359937fab1a565422d1d6d8de347f2e4e12b7782
Parents: aaf4551
Author: Sushrut Meghshyam Ikhar <su...@gmail.com>
Authored: Tue Jun 6 17:53:27 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Tue Jun 6 17:53:27 2017 +0530
----------------------------------------------------------------------
.../lens/cube/parse/ExpressionResolver.java | 4 --
.../lens/cube/parse/StorageCandidate.java | 8 +++-
.../cube/parse/MockCompletenessChecker.java | 10 ++--
.../lens/cube/parse/TestCubeRewriter.java | 49 +++++++++++++++++++-
.../resources/schema/cubes/base/basecube.xml | 5 ++
.../resources/schema/cubes/derived/der1.xml | 1 +
.../schema/facts/testfact2_raw_base.xml | 1 +
.../schema/facts/virtual/virtualfact.xml | 27 +++++++++++
.../test/resources/schema/facts/virtualfact.xml | 27 -----------
9 files changed, 95 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/359937fa/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index ea6d5c7..b1654d1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -556,12 +556,8 @@ class ExpressionResolver implements ContextRewriter {
log.info("Removing expression {} as all tables have non reachable fields", esc);
iterator.remove();
removedEsc.add(esc);
- removed = true;
break;
}
- if (removed) {
- continue;
- }
//remove expressions which are not valid in the timerange queried
// If an expression is defined as
// ex = a + b // from t1 to t2;
http://git-wip-us.apache.org/repos/asf/lens/blob/359937fa/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 3f5e2e5..7980797 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -651,6 +651,10 @@ public class StorageCandidate implements Candidate, CandidateTable {
}
private boolean evaluateMeasuresCompleteness(TimeRange timeRange) throws LensException {
+ if (getCubeMetastoreClient() == null || !getCubeMetastoreClient().isDataCompletenessCheckEnabled()) {
+ log.info("Skipping availability check for the fact table: {} as dataCompleteness check is not enabled", fact);
+ return true;
+ }
String factDataCompletenessTag = fact.getDataCompletenessTag();
if (factDataCompletenessTag == null) {
log.info("Not checking completeness for the fact table:{} as the dataCompletenessTag is not set", fact);
@@ -673,7 +677,8 @@ public class StorageCandidate implements Candidate, CandidateTable {
log.info("No Queried measures with the dataCompletenessTag, hence skipping the availability check");
return true;
}
- boolean isDataComplete = false;
+ // default completenessTag will be true
+ boolean isDataComplete = true;
DataCompletenessChecker completenessChecker = getCubeMetastoreClient().getCompletenessChecker();
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -695,6 +700,7 @@ public class StorageCandidate implements Candidate, CandidateTable {
String measureorExprFromTag = tagToMeasureOrExprMap.get(tag);
dataCompletenessMap.computeIfAbsent(measureorExprFromTag, k -> new HashMap<>())
.put(formatter.format(completenessResult.getKey()), completenessResult.getValue());
+ // set completeness to false if availability for measure is below threshold
isDataComplete = false;
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/359937fa/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java
index 76e81d5..c1936eb 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/MockCompletenessChecker.java
@@ -33,10 +33,12 @@ public class MockCompletenessChecker implements DataCompletenessChecker {
Calendar cal = Calendar.getInstance();
cal.setTimeZone(TimeZone.getTimeZone("GMT"));
cal.add(Calendar.DATE, -1);
- if (factTag.equals("f1")) {
- partitionCompleteness.put(cal.getTime(), 80f);
- } else {
- partitionCompleteness.put(cal.getTime(), 90f);
+ if (start.before(cal.getTime()) && end.after(cal.getTime())) {
+ if (factTag.equals("f1")) {
+ partitionCompleteness.put(cal.getTime(), 80f);
+ } else {
+ partitionCompleteness.put(cal.getTime(), 90f);
+ }
}
result.put("tag1", partitionCompleteness);
return result;
http://git-wip-us.apache.org/repos/asf/lens/blob/359937fa/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 65ca861..d8c7335 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -981,7 +981,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
conf.setStrings(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL, "dt");
String hqlQuery = rewrite("select SUM(msr9) from basecube where " + TWO_DAYS_RANGE, conf);
String expected = getExpectedQuery("basecube", "select sum(basecube.msr9) as `sum(msr9)` FROM ", null, null,
- getWhereForHourly2days("basecube", "c1_testfact5_raw_base"));
+ getWhereForHourly2days("basecube", "c1_testfact5_raw_base"));
compareQueries(hqlQuery, expected);
}
@@ -1007,6 +1007,53 @@ public class TestCubeRewriter extends TestQueryRewrite {
}
}
+ /*
+ * The test is to check that query is rewritten successfully if there is missing entry in
+ * dataavailability service for the measure's tag
+ */
+ @Test
+ public void testQueryWithMeasureWithDataCompletenessTagWithDataAvailiability() throws ParseException,
+ LensException {
+ NoCandidateFactAvailableException ne;
+ PruneCauses.BriefAndDetailedError pruneCauses;
+ String hqlQuery;
+ String expected;
+
+ Configuration conf = getConf();
+ conf.setStrings(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL, "dt");
+
+ conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true);
+
+ // 1. data completeness check applicable
+ ne = getLensExceptionInRewrite("select SUM(msr16) from basecube" + " where " + TWO_DAYS_RANGE, conf);
+ pruneCauses = ne.getJsonMessage();
+ assertEquals(pruneCauses.getBrief().substring(0, 10), INCOMPLETE_PARTITION.errorFormat.substring(0, 10),
+ pruneCauses.getBrief());
+
+ // 2. time outside data completeness check but missing partitions
+ ne = getLensExceptionInRewrite("select SUM(msr16) from basecube where " + TWO_DAYS_RANGE_BEFORE_4_DAYS, conf);
+ pruneCauses = ne.getJsonMessage();
+ assertEquals(pruneCauses.getBrief().substring(0, 10), MISSING_PARTITIONS.errorFormat.substring(0, 10),
+ pruneCauses.getBrief());
+
+
+ conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false);
+
+ // 3. query allowed on partial data although data incomplete
+ hqlQuery = rewrite("select SUM(msr16) from basecube" + " where " + TWO_DAYS_RANGE, conf);
+ expected = getExpectedQuery("basecube", "select sum(basecube.msr16) as `sum(msr16)` FROM ", null, null,
+ getWhereForHourly2days("basecube", "c1_testfact2_raw_base"));
+ compareQueries(hqlQuery.toLowerCase(), expected.toLowerCase());
+
+ // 4. query allowed on partial data with missing partitions but outside data availability window
+ hqlQuery = rewrite("select SUM(msr16) from basecube" + " where " + TWO_DAYS_RANGE_BEFORE_4_DAYS, conf);
+ expected = getExpectedQuery("basecube", "select sum(basecube.msr16) as `sum(msr16)` FROM ", null, null,
+ getWhereForUpdatePeriods("basecube", "c1_testfact2_raw_base",
+ DateUtils.addHours(getDateWithOffset(UpdatePeriod.DAILY, -6), -1),
+ getDateWithOffset(UpdatePeriod.DAILY, -4), Sets.newHashSet(UpdatePeriod.HOURLY)));
+ compareQueries(hqlQuery.toLowerCase(), expected.toLowerCase());
+ }
+
@Test
public void testCubeWhereQueryForMonthWithNoPartialData() throws Exception {
Configuration conf = getConf();
http://git-wip-us.apache.org/repos/asf/lens/blob/359937fa/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
index 708b510..bcea938 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
@@ -90,6 +90,11 @@
</measure>
<measure _type="INT" name="segmsr1" default_aggr="SUM" description="segmentation measure 1"
display_string="segmentation measure 1"/>
+ <measure _type="BIGINT" name="msr16" description="sixteenth measure">
+ <tags>
+ <property name="cube.measure.datacompleteness.tag" value="tag1"/>
+ </tags>
+ </measure>
</measures>
<dim_attributes>
<dim_attribute _type="string" name="business" display_string="business unit"
http://git-wip-us.apache.org/repos/asf/lens/blob/359937fa/lens-cube/src/test/resources/schema/cubes/derived/der1.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/derived/der1.xml b/lens-cube/src/test/resources/schema/cubes/derived/der1.xml
index cf36cc7..01bec03 100644
--- a/lens-cube/src/test/resources/schema/cubes/derived/der1.xml
+++ b/lens-cube/src/test/resources/schema/cubes/derived/der1.xml
@@ -33,6 +33,7 @@
<measure_name>msr1</measure_name>
<measure_name>msr11</measure_name>
<measure_name>msr9</measure_name>
+ <measure_name>msr16</measure_name>
</measure_names>
<dim_attr_names>
<attr_name>dim1</attr_name>
http://git-wip-us.apache.org/repos/asf/lens/blob/359937fa/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml b/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml
index 0bab461..69fd8d9 100644
--- a/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml
+++ b/lens-cube/src/test/resources/schema/facts/testfact2_raw_base.xml
@@ -24,6 +24,7 @@
<column name="msr11" _type="int" comment="first measure"/>
<column name="msr12" _type="float" comment="second measure"/>
<column name="msr9" _type="bigint" comment="ninth measure"/>
+ <column name="msr16" _type="bigint" comment="sixteenth measure"/>
<column name="d_time" _type="timestamp" comment="event time"/>
<column name="processing_time" _type="timestamp" comment="processing time"/>
<column name="dim1" _type="string" comment="base dim"/>
http://git-wip-us.apache.org/repos/asf/lens/blob/359937fa/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml b/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
new file mode 100644
index 0000000..860965f
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_virtual_fact_table source_fact_name="summary1" cube_name="virtualcube" name="virtualfact1" xmlns="uri:lens:cube:0.1"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <properties>
+ <property name="cube.fact.query.where.filter" value=" dim1 = 10 "/>
+ </properties>
+</x_virtual_fact_table>
http://git-wip-us.apache.org/repos/asf/lens/blob/359937fa/lens-cube/src/test/resources/schema/facts/virtualfact.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/virtualfact.xml b/lens-cube/src/test/resources/schema/facts/virtualfact.xml
deleted file mode 100644
index 860965f..0000000
--- a/lens-cube/src/test/resources/schema/facts/virtualfact.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<x_virtual_fact_table source_fact_name="summary1" cube_name="virtualcube" name="virtualfact1" xmlns="uri:lens:cube:0.1"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
- <properties>
- <property name="cube.fact.query.where.filter" value=" dim1 = 10 "/>
- </properties>
-</x_virtual_fact_table>