You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by ra...@apache.org on 2018/02/06 05:46:44 UTC
[28/50] lens git commit: LENS-1466 : Fix for 'Rewritten queries not
resolving time_range_in clause'
LENS-1466 : Fix for 'Rewritten queries not resolving time_range_in clause'
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ff4f3726
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ff4f3726
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ff4f3726
Branch: refs/heads/master
Commit: ff4f3726a25db684ff7623fea68e954712dd925f
Parents: 10eef27
Author: Rajitha R <ra...@gmail.com>
Authored: Fri Sep 1 13:02:32 2017 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Sep 1 15:23:45 2017 +0530
----------------------------------------------------------------------
.../lens/cube/parse/StorageCandidate.java | 2 +-
.../apache/lens/cube/metadata/DateFactory.java | 6 ++
.../apache/lens/cube/parse/CubeTestSetup.java | 47 +++++++++++--
.../lens/cube/parse/TestCubeRewriter.java | 15 ++++-
.../resources/schema/cubes/base/testcube.xml | 2 +
.../test/resources/schema/facts/summary5.xml | 70 ++++++++++++++++++++
.../src/test/resources/schema/storages/c98.xml | 26 ++++++++
7 files changed, 159 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 2b1add3..27835b3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -533,9 +533,9 @@ public class StorageCandidate implements Candidate, CandidateTable {
log.debug("Adding non existing partition {}", part);
if (addNonExistingParts) {
// Add non existing partitions for all cases of whether we populate all non existing or not.
- this.participatingUpdatePeriods.add(maxInterval);
missingPartitions.add(part);
if (!failOnPartialData) {
+ this.participatingUpdatePeriods.add(maxInterval);
partitions.add(part);
part.getStorageTables().add(storageTable);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
index 855f54a..75f2abd 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
@@ -150,6 +150,7 @@ public class DateFactory {
// Time Instances as Date Type
public static final Date NOW;
+ public static final Date ONEDAY_BACK;
public static final Date TWODAYS_BACK;
public static final Date TWO_MONTHS_BACK;
public static final Date THIS_MONTH_TRUNCATED;
@@ -168,6 +169,7 @@ public class DateFactory {
public static final String TWO_DAYS_RANGE_TTD2;
public static final String TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS;
public static final String TWO_DAYS_RANGE_IT;
+ public static final String ONE_DAY_RANGE_IT;
public static final String THIS_YEAR_RANGE;
public static final String LAST_YEAR_RANGE;
public static final String TWO_MONTHS_RANGE_UPTO_MONTH;
@@ -189,6 +191,9 @@ public class DateFactory {
// Figure out if current hour is 0th hour
zerothHour = getDateStringWithOffset(HOURLY, 0).endsWith("-00");
+ ONEDAY_BACK = getDateWithOffset(DAILY, -1);
+ System.out.println("Test ONEDAY_BACK:" + ONEDAY_BACK);
+
TWODAYS_BACK = getDateWithOffset(DAILY, -2);
System.out.println("Test TWODAYS_BACK:" + TWODAYS_BACK);
@@ -214,6 +219,7 @@ public class DateFactory {
TWO_DAYS_RANGE_TTD2 = getTimeRangeString("test_time_dim2", DAILY, -2, 0, HOURLY);
TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS = getTimeRangeString("test_time_dim2", DAILY, -6, -4, HOURLY);
TWO_DAYS_RANGE_IT = getTimeRangeString("it", DAILY, -2, 0, HOURLY);
+ ONE_DAY_RANGE_IT = getTimeRangeString("it", DAILY, -1, 0, DAILY);
THIS_YEAR_RANGE = getTimeRangeString(YEARLY, 0, 1);
LAST_YEAR_RANGE = getTimeRangeString(YEARLY, -1, 0);
TWO_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -2, 0);
http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 0366e56..5608b39 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -22,12 +22,7 @@ package org.apache.lens.cube.parse;
import static java.util.Calendar.DAY_OF_MONTH;
import static java.util.Calendar.HOUR_OF_DAY;
-import static org.apache.lens.cube.metadata.DateFactory.BEFORE_4_DAYS;
-import static org.apache.lens.cube.metadata.DateFactory.BEFORE_6_DAYS;
-import static org.apache.lens.cube.metadata.DateFactory.NOW;
-import static org.apache.lens.cube.metadata.DateFactory.TWODAYS_BACK;
-import static org.apache.lens.cube.metadata.DateFactory.TWO_MONTHS_BACK;
-import static org.apache.lens.cube.metadata.DateFactory.isZerothHour;
+import static org.apache.lens.cube.metadata.DateFactory.*;
import static org.apache.lens.cube.metadata.UpdatePeriod.DAILY;
import static org.apache.lens.cube.metadata.UpdatePeriod.HOURLY;
import static org.apache.lens.cube.metadata.UpdatePeriod.MINUTELY;
@@ -132,6 +127,7 @@ public class CubeTestSetup {
private static String c3 = "C3";
private static String c4 = "C4";
private static String c5 = "C5";
+ private static String c98 = "C98";
private static String c99 = "C99";
private static Map<String, String> factValidityProperties = Maps.newHashMap();
@Getter
@@ -881,10 +877,13 @@ public class CubeTestSetup {
CubeFactTable fact3 = client.getCubeFactTable(factName);
createPIEParts(client, fact3, c2);
-
factName = "summary4";
CubeFactTable fact4 = client.getCubeFactTable(factName);
createPIEParts(client, fact4, c2);
+
+ factName = "summary5";
+ CubeFactTable fact5 = client.getCubeFactTable(factName);
+ createPIParts(client, fact5, c98);
}
private void createBaseCubeFactPartitions(CubeMetastoreClient client) throws HiveException, LensException {
@@ -994,6 +993,7 @@ public class CubeTestSetup {
pTimes.get(HOURLY).add(ptime);
iTimes.get(HOURLY).add(itime);
client.addPartition(sPartSpec, storageName, CubeTableType.FACT);
+
pcal.add(HOUR_OF_DAY, 1);
ical.add(HOUR_OF_DAY, 1);
}
@@ -1015,6 +1015,39 @@ public class CubeTestSetup {
}
}
+
+ private void createPIParts(CubeMetastoreClient client, CubeFactTable fact, String storageName)
+ throws Exception {
+ // Add partitions in PI storage
+ //daily partition registered for pt=day1, it = day1
+ //hourly partitions registered for pt=day1-hours[0-23] it=day1-hours[0-23]
+ Calendar pcal = Calendar.getInstance();
+ pcal.setTime(ONEDAY_BACK);
+ pcal.set(HOUR_OF_DAY, 0);
+ Calendar ical = Calendar.getInstance();
+ ical.setTime(ONEDAY_BACK);
+ ical.set(HOUR_OF_DAY, 0);
+
+ Date ptime = pcal.getTime();
+ Date itime = ical.getTime();
+ Map<String, Date> timeParts = new HashMap<String, Date>();
+
+ timeParts.put("pt", ptime);
+ timeParts.put("it", itime);
+ StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, DAILY);
+ client.addPartition(sPartSpec, storageName, CubeTableType.FACT);
+ for (int i = 0; i < 24; i++) {
+ ptime = pcal.getTime();
+ itime = ical.getTime();
+ timeParts.put("pt", ptime);
+ timeParts.put("it", itime);
+ sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
+ client.addPartition(sPartSpec, storageName, CubeTableType.FACT);
+ pcal.add(HOUR_OF_DAY, 1);
+ ical.add(HOUR_OF_DAY, 1);
+ }
+ }
+
public static void printQueryAST(String query, String label) throws LensException {
System.out.println("--" + label + "--AST--");
System.out.println("--query- " + query);
http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index bc0fa2e..9b54957 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -65,7 +65,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
@BeforeTest
public void setupDriver() throws Exception {
conf = LensServerAPITestUtil.getConfiguration(
- DRIVER_SUPPORTED_STORAGES, "C0,C1,C2",
+ DRIVER_SUPPORTED_STORAGES, "C0,C1,C2,C98",
DISABLE_AUTO_JOINS, true,
ENABLE_SELECT_TO_GROUPBY, true,
ENABLE_GROUP_BY_TO_SELECT, true,
@@ -1470,6 +1470,19 @@ public class TestCubeRewriter extends TestQueryRewrite {
}
@Test
+ public void testTimeRangeIn() throws Exception {
+ //check whether time_range_in is resolving in cube rewrite
+ Configuration conf = getConf();
+ conf.set(CubeQueryConfUtil.PROCESS_TIME_PART_COL, "pt");
+ conf.set(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "true");
+ conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, AbridgedTimeRangeWriter.class, TimeRangeWriter.class);
+ CubeQueryContext ctx = rewriteCtx("select dim1, sum(msr23)" + " from testCube" + " where " + ONE_DAY_RANGE_IT,
+ conf);
+ String rewrittenQuery = ctx.toHQL();
+ assertTrue(!rewrittenQuery.contains("time_range_in"));
+ }
+
+ @Test
public void testCubeQueryWithMultipleRanges() throws Exception {
String hqlQuery =
rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE + " OR "
http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
index 9d2bb02..4cafd31 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
@@ -32,6 +32,8 @@
<property name="cube.timedim.partition.test_time_dim2" value="ttd2"/>
</properties>
<measures>
+ <measure _type="FLOAT" default_aggr="SUM" unit="RS" name="msr23" display_string="Measure23"
+ description="twenty third measure"/>
<measure _type="FLOAT" default_aggr="SUM" unit="RS" name="msr21" display_string="Measure22"
description="second measure"/>
<measure _type="BIGINT" default_aggr="COUNT" name="msr4" display_string="Measure4" description="fourth measure"/>
http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/resources/schema/facts/summary5.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/summary5.xml b/lens-cube/src/test/resources/schema/facts/summary5.xml
new file mode 100644
index 0000000..a4bbf71
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/summary5.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_fact_table name="summary5" cube_name="testCube" weight="15.0" xmlns="uri:lens:cube:0.1">
+ <columns>
+ <column name="msr23" _type="float" comment="twenty third measure"/>
+ <column name="dim1" _type="string" comment="dim1"/>
+ <column name="dim2" _type="string" comment="dim2"/>
+ </columns>
+ <properties>
+ <property name="cube.fact.relative.start.time" value="now.year - 90 days"/>
+ <property name="cube.table.summary5.weight" value="15.0"/>
+ <property name="cube.fact.summary5.storages" value="C98"/>
+ <property name="cube.fact.summary5.c98.updateperiods" value="HOURLY,DAILY"/>
+ </properties>
+ <storage_tables>
+ <storage_table>
+ <storage_name>C98</storage_name>
+ <update_periods>
+ <update_period_table_descriptor>
+ <update_period>HOURLY</update_period>
+ <table_desc external="false">
+ <part_cols>
+ <column name="pt" _type="string" comment="date partition"/>
+ <column name="it" _type="string" comment="date partition"/>
+ </part_cols>
+ <table_parameters>
+ <property name="cube.storagetable.start.times" value="now -4 days"/>
+ </table_parameters>
+ <time_part_cols>pt</time_part_cols>
+ <time_part_cols>it</time_part_cols>
+ </table_desc>
+ </update_period_table_descriptor>
+ <update_period_table_descriptor>
+ <update_period>DAILY</update_period>
+ <table_desc external="false">
+ <part_cols>
+ <column name="pt" _type="string" comment="date partition"/>
+ <column name="it" _type="string" comment="date partition"/>
+ </part_cols>
+ <table_parameters>
+ <property name="cube.storagetable.start.times" value="now.month -3 months"/>
+ </table_parameters>
+ <time_part_cols>pt</time_part_cols>
+ <time_part_cols>it</time_part_cols>
+ </table_desc>
+ </update_period_table_descriptor>
+ </update_periods>
+ </storage_table>
+ </storage_tables>
+</x_fact_table>
+
http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/resources/schema/storages/c98.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/storages/c98.xml b/lens-cube/src/test/resources/schema/storages/c98.xml
new file mode 100644
index 0000000..550b421
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/storages/c98.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_storage name="c98" classname="org.apache.lens.cube.metadata.HDFSStorage" xmlns="uri:lens:cube:0.1">
+ <properties>
+ <property name="cube.table.c98.weight" value="0.0"/>
+ </properties>
+</x_storage>
\ No newline at end of file