You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by ra...@apache.org on 2018/08/16 06:49:58 UTC
[1/2] lens git commit: LENS-1525 : Adding start and end times in cube
Repository: lens
Updated Branches:
refs/heads/master e39dec5f8 -> 5a73df0d2
LENS-1525 : Adding start and end times in cube
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c4cad0af
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c4cad0af
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c4cad0af
Branch: refs/heads/master
Commit: c4cad0af68f180daaf4e0abffb4961aa109ca038
Parents: e39dec5
Author: Rajitha R <ra...@apache.org>
Authored: Thu Aug 16 12:19:14 2018 +0530
Committer: Rajitha.R <ra...@IM0318-L0.corp.inmobi.com>
Committed: Thu Aug 16 12:19:14 2018 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/cube-0.1.xsd | 15 +++++++
.../lens/cube/error/LensCubeErrorCode.java | 1 +
.../org/apache/lens/cube/metadata/Cube.java | 36 ++++++++++++++++
.../lens/cube/metadata/CubeInterface.java | 44 ++++++++++++++++++++
.../apache/lens/cube/metadata/DerivedCube.java | 31 ++++++++++++++
.../lens/cube/metadata/MetastoreConstants.java | 4 ++
.../lens/cube/parse/TimerangeResolver.java | 7 +++-
.../apache/lens/cube/metadata/DateFactory.java | 2 +
.../lens/cube/parse/TestTimeRangeResolver.java | 11 +++++
.../resources/schema/cubes/base/testcube.xml | 2 +
10 files changed, 152 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index 4d10d24..65bd6ea 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -102,6 +102,21 @@
corresponding partition column different in the underlying storage tables. This base cube's property gets
propagated to its derived cubes.
+ 3. cube.absolute.start.time: start time of the cube. For queries that ask for time before this,
+ this cube throws an exception. Time format can be as you would specify in the time_range_in clause.
+ i.e. yyyy[-mm[-dd[-hh[:MM[:ss[,SSS]]]]]]
+
+ 4. cube.relative.start.time: Here you can specify cube's relative validity relative to current time.
+ Useful if you want to specify e.g. this cube is valid for today - 90 days. Can be specified as just
+ a time difference e.g. "-90 days". Or can be specified in relative syntax.
+ e.g. now.year or now.day - 6 hour etc.
+
+ 5. cube.absolute.end.time: This is to restrict the end date users can query on this cube.
+ Format same as absolute start time.
+
+ 6. cube.relative.end.time: You can specify the end date for cube
+ relative to current date. This can be specified as just a time difference e.g. "+90 days"
+
The properties that are over-ridable are
1. cube.allfields.queriable - (defaults to true) - If set to true, all the fields (measures, attributes) are
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
index ed076e2..49343a7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
@@ -40,6 +40,7 @@ public enum LensCubeErrorCode {
FROM_AFTER_TO(3015, 0),
JOIN_TARGET_NOT_CUBE_TABLE(3016, 0),
ORDERBY_ALIAS_CONTAINING_WHITESPACE(3036, 0),
+ QUERY_OUT_OF_ALLOWED_RANGE(3037, 0),
// Error codes different for drivers
CANNOT_USE_TIMERANGE_WRITER(3017, 100),
NO_DEFAULT_AGGREGATE(3018, 200),
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java
index 7c3da2c..1d85fbd 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java
@@ -24,6 +24,7 @@ import java.util.*;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
@@ -414,4 +415,39 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
return partCol;
}
+ @Override
+ public Date getAbsoluteStartTime() {
+ return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.CUBE_ABSOLUTE_START_TIME),
+ false, true);
+ }
+
+ @Override
+ public Date getRelativeStartTime() {
+ return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.CUBE_RELATIVE_START_TIME),
+ true, true);
+ }
+
+ @Override
+ public Date getStartTime() {
+ return Collections.max(Lists.newArrayList(getRelativeStartTime(), getAbsoluteStartTime()));
+ }
+
+ @Override
+ public Date getAbsoluteEndTime() {
+ return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.CUBE_ABSOLUTE_END_TIME),
+ false, false);
+ }
+
+ @Override
+ public Date getRelativeEndTime() {
+ return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.CUBE_RELATIVE_END_TIME),
+ true, false);
+ }
+
+ @Override
+ public Date getEndTime() {
+ return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime()));
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java
index 6477a6b..3f2b670 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.cube.metadata;
+import java.util.Date;
import java.util.Set;
/**
@@ -155,4 +156,47 @@ public interface CubeInterface extends Named {
* @return true or false
*/
boolean allFieldsQueriable();
+
+ /**
+ * Absolute start time of the cube
+ *
+ * @return Absolute Start time of the cube {@link Date}
+ */
+ Date getAbsoluteStartTime();
+
+ /**
+ * Relative start time of the cube
+ *
+ * @return Relative Start time of the cube {@link Date}
+ */
+ Date getRelativeStartTime();
+
+ /**
+ * Start time of the cube
+ *
+ * @return Start time of the cube {@link Date}
+ */
+ Date getStartTime();
+
+ /**
+ * Absolute end time of the cube
+ *
+ * @return Absolute End time of the cube {@link Date}
+ */
+ Date getAbsoluteEndTime();
+
+ /**
+ * Relative End time of the cube
+ *
+ * @return Relative end time of the cube {@link Date}
+ */
+ Date getRelativeEndTime();
+
+ /**
+ * End time of the cube
+ *
+ * @return End time of the cube {@link Date}
+ */
+ Date getEndTime();
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java
index 4c73785..2ff4ab1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java
@@ -338,4 +338,35 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface {
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public Date getAbsoluteStartTime() {
+ return parent.getAbsoluteStartTime();
+ }
+
+ @Override
+ public Date getRelativeStartTime() {
+ return parent.getRelativeStartTime();
+ }
+
+ @Override
+ public Date getStartTime() {
+ return parent.getStartTime();
+ }
+
+ @Override
+ public Date getAbsoluteEndTime() {
+ return parent.getAbsoluteEndTime();
+ }
+
+ @Override
+ public Date getRelativeEndTime() {
+ return parent.getRelativeEndTime();
+ }
+
+ @Override
+ public Date getEndTime() {
+ return parent.getEndTime();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
index 160addb..5bdfea4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
@@ -39,6 +39,10 @@ public final class MetastoreConstants {
public static final String TIMED_DIMENSIONS_LIST_SFX = ".timed.dimensions.list";
public static final String PARENT_CUBE_SFX = ".parent.cube";
public static final String CUBE_ALL_FIELDS_QUERIABLE = "cube.allfields.queriable";
+ public static final String CUBE_ABSOLUTE_START_TIME = "cube.absolute.start.time";
+ public static final String CUBE_RELATIVE_START_TIME = "cube.relative.start.time";
+ public static final String CUBE_ABSOLUTE_END_TIME = "cube.absolute.end.time";
+ public static final String CUBE_RELATIVE_END_TIME = "cube.relative.end.time";
// Uber dimension constants
public static final String DIMENSION_PFX = "dimension.";
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java
index 8e77a82..de5214f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java
@@ -45,9 +45,14 @@ class TimerangeResolver implements ContextRewriter {
return;
}
extractTimeRange(cubeql);
+ CubeInterface cube = cubeql.getCube();
+ for(TimeRange timeRange : cubeql.getTimeRanges()) {
+ if (!(timeRange.getFromDate().after(cube.getStartTime()) && timeRange.getToDate().before(cube.getEndTime()))) {
+ throw new LensException(LensCubeErrorCode.QUERY_OUT_OF_ALLOWED_RANGE.getLensErrorInfo());
+ }
+ }
}
-
private void extractTimeRange(CubeQueryContext cubeql) throws LensException {
// get time range -
// Time range should be direct child of where condition
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
index c4404f1..99c5961 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
@@ -173,6 +173,7 @@ public class DateFactory {
public static final String ONE_DAY_RANGE_IT;
public static final String THIS_YEAR_RANGE;
public static final String LAST_YEAR_RANGE;
+ public static final String PREV_FIVE_TO_NEXT_FIVE_YEAR_RANGE;
public static final String TWO_MONTHS_RANGE_UPTO_MONTH;
public static final String TWO_MONTHS_RANGE_UPTO_DAYS;
public static final String TWO_MONTHS_RANGE_UPTO_HOURS;
@@ -224,6 +225,7 @@ public class DateFactory {
ONE_DAY_RANGE_IT = getTimeRangeString("it", DAILY, -1, 0, DAILY);
THIS_YEAR_RANGE = getTimeRangeString(YEARLY, 0, 1);
LAST_YEAR_RANGE = getTimeRangeString(YEARLY, -1, 0);
+ PREV_FIVE_TO_NEXT_FIVE_YEAR_RANGE = getTimeRangeString(YEARLY, -5, 5);
TWO_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -2, 0);
TWO_MONTHS_RANGE_UPTO_DAYS = getTimeRangeString(MONTHLY, -2, 0, DAILY);
TWO_MONTHS_RANGE_UPTO_HOURS = getTimeRangeString(MONTHLY, -2, 0, HOURLY);
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java
index 181608f..632d941 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java
@@ -42,6 +42,7 @@ import org.testng.annotations.Test;
public class TestTimeRangeResolver extends TestQueryRewrite {
private Configuration conf;
+ Date now = new Date();
@BeforeTest
public void setupDriver() throws Exception {
@@ -102,6 +103,14 @@ public class TestTimeRangeResolver extends TestQueryRewrite {
assertEquals(e.getMessage(), "NO_CANDIDATE_FACT_AVAILABLE[Range not answerable]");
}
+ @Test
+ public void testQueryTimeRange() throws LensException {
+ String query = "select msr2 from " + CubeTestSetup.TEST_CUBE_NAME + " where " + PREV_FIVE_TO_NEXT_FIVE_YEAR_RANGE;
+
+ LensException e = getLensExceptionInRewrite(query, getConf());
+ assertEquals(e.getErrorInfo().getErrorName(), "QUERY_OUT_OF_ALLOWED_RANGE");
+ }
+
/**
*
* @param stoargeName storageName_factName
@@ -119,4 +128,6 @@ public class TestTimeRangeResolver extends TestQueryRewrite {
}
return new ArrayList<>();
}
+
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cad0af/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
index 4cafd31..3d517ee 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
@@ -21,6 +21,8 @@
-->
<x_base_cube name="testcube" xmlns="uri:lens:cube:0.1">
<properties>
+ <property name="cube.relative.start.time" value="now.year - 2 years"/>
+ <property name="cube.relative.end.time" value="now.day + 2 years"/>
<property name="cube.timedim.partition.et" value="et"/>
<property name="cube.timedim.partition.it" value="it"/>
<property name="cube.timedim.partition.d_time" value="dt"/>
[2/2] lens git commit: LENS-1525 : Adding start and end times in cube
Posted by ra...@apache.org.
LENS-1525 : Adding start and end times in cube
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5a73df0d
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5a73df0d
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5a73df0d
Branch: refs/heads/master
Commit: 5a73df0d26e1c69d77bf43a2b5ef06559b4032d4
Parents: c4cad0a
Author: Rajitha R <ra...@apache.org>
Authored: Thu Aug 16 12:19:19 2018 +0530
Committer: Rajitha.R <ra...@IM0318-L0.corp.inmobi.com>
Committed: Thu Aug 16 12:19:19 2018 +0530
----------------------------------------------------------------------
.../cube/metadata/TestCubeStartAndEndTimes.java | 105 +++++++++++++++++++
1 file changed, 105 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/5a73df0d/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeStartAndEndTimes.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeStartAndEndTimes.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeStartAndEndTimes.java
new file mode 100644
index 0000000..2f895b8
--- /dev/null
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeStartAndEndTimes.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lens.server.api.error.LensException;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class TestCubeStartAndEndTimes {
+
+ Date now = new Date();
+
+ @DataProvider(name = "properties")
+ public Object[][] factProperties() throws LensException {
+ String minus1DaysRelative = "now.day -1 days";
+ String minus2DaysRelative = "now.day -2 days";
+ String plus1DaysRelative = "now.day +1 days";
+ String plus2DaysRelative = "now.day +2 days";
+
+ String minus1DaysAbsolute = DateUtil.relativeToAbsolute(minus1DaysRelative, now);
+ String minus2DaysAbsolute = DateUtil.relativeToAbsolute(minus2DaysRelative, now);
+ String plus1DaysAbsolute = DateUtil.relativeToAbsolute(plus1DaysRelative, now);
+ String plus2DaysAbsolute = DateUtil.relativeToAbsolute(plus2DaysRelative, now);
+
+ Date minus1DaysDate = DateUtil.resolveRelativeDate(minus1DaysRelative, now);
+ Date minus2DaysDate = DateUtil.resolveRelativeDate(minus2DaysRelative, now);
+ Date plus1DaysDate = DateUtil.resolveRelativeDate(plus1DaysRelative, now);
+ Date plus2DaysDate = DateUtil.resolveRelativeDate(plus2DaysRelative, now);
+
+ return new Object[][]{
+ {null, null, null, null, new Date(Long.MIN_VALUE), new Date(Long.MAX_VALUE)},
+ {null, minus2DaysRelative, null, plus2DaysRelative, minus2DaysDate, plus2DaysDate},
+ {minus2DaysAbsolute, null, plus2DaysAbsolute, null, minus2DaysDate, plus2DaysDate},
+ {minus1DaysAbsolute, minus2DaysRelative, plus1DaysAbsolute, plus2DaysRelative, minus1DaysDate, plus1DaysDate},
+ {minus2DaysAbsolute, minus1DaysRelative, plus2DaysAbsolute, plus1DaysRelative, minus1DaysDate, plus1DaysDate},
+ };
+ }
+ private Cube getMockCube(Map<String, String> properties) {
+ Cube cube = mock(Cube.class);
+
+ when(cube.now()).thenReturn(now);
+
+ when(cube.getProperties()).thenReturn(properties);
+
+ when(cube.getRelativeStartTime()).thenCallRealMethod();
+ when(cube.getAbsoluteStartTime()).thenCallRealMethod();
+
+ when(cube.getRelativeEndTime()).thenCallRealMethod();
+ when(cube.getAbsoluteEndTime()).thenCallRealMethod();
+
+ when(cube.getStartTime()).thenCallRealMethod();
+ when(cube.getEndTime()).thenCallRealMethod();
+
+ return cube;
+ }
+
+ @Test(dataProvider = "properties")
+ public void testStartAndEndTime(String absoluteStartProperty, String relativeStartProperty,
+ String absoluteEndProperty, String relativeEndProperty,
+ Date expectedStartTime, Date expectedEndTime) throws Exception {
+
+ Map<String, String> properties = new HashMap<>();
+ if (absoluteStartProperty != null) {
+ properties.put(MetastoreConstants.CUBE_ABSOLUTE_START_TIME, absoluteStartProperty);
+ }
+ if (relativeStartProperty != null) {
+ properties.put(MetastoreConstants.CUBE_RELATIVE_START_TIME, relativeStartProperty);
+ }
+ if (absoluteEndProperty != null) {
+ properties.put(MetastoreConstants.CUBE_ABSOLUTE_END_TIME, absoluteEndProperty);
+ }
+ if (relativeEndProperty != null) {
+ properties.put(MetastoreConstants.CUBE_RELATIVE_END_TIME, relativeEndProperty);
+ }
+ Cube cube = getMockCube(properties);
+
+ assertEquals(cube.getStartTime(), expectedStartTime);
+ assertEquals(cube.getEndTime(), expectedEndTime);
+ }
+}