You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by ra...@apache.org on 2015/10/09 06:17:12 UTC
[07/50] [abbrv] lens git commit: LENS-745 : Fix dropping latest
partition from timeline
LENS-745 : Fix dropping latest partition from timeline
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/86f251a7
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/86f251a7
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/86f251a7
Branch: refs/heads/current-release-line
Commit: 86f251a75b3a010a795f29be982ee19d9b2ba780
Parents: 2901b4b
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Aug 31 10:19:56 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Aug 31 10:19:56 2015 +0530
----------------------------------------------------------------------
.../lens/cube/metadata/TimePartition.java | 2 -
.../apache/lens/cube/metadata/UpdatePeriod.java | 57 ++++++--
.../timeline/StoreAllPartitionTimeline.java | 5 +-
.../org/apache/lens/cube/parse/DateUtil.java | 77 ++++------
.../org/apache/lens/cube/parse/TimeRange.java | 2 +-
.../cube/metadata/TestCubeMetastoreClient.java | 21 +--
.../lens/cube/metadata/TestTimePartition.java | 64 +++++----
.../lens/cube/metadata/UpdatePeriodTest.java | 60 ++++++++
.../timeline/TestPartitionTimelines.java | 143 +++++++++++--------
.../apache/lens/cube/parse/TestDateUtil.java | 4 +-
10 files changed, 258 insertions(+), 177 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
index 11e6ef6..d52f168 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
@@ -37,8 +37,6 @@ public class TimePartition implements Comparable<TimePartition>, Named {
private TimePartition(@NonNull UpdatePeriod updatePeriod, @NonNull Date date) {
this.updatePeriod = updatePeriod;
- Calendar cal = Calendar.getInstance();
- cal.setTime(date);
this.date = updatePeriod.truncate(date);
this.dateString = updatePeriod.format().format(this.date);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
index b4dff46..6c61e94 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
@@ -27,22 +27,28 @@ import java.util.Calendar;
import java.util.Comparator;
import java.util.Date;
+import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.cube.parse.DateUtil;
+import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang3.time.DateUtils;
+import lombok.Getter;
+
public enum UpdatePeriod implements Named {
- SECONDLY(SECOND, 1000, 1.4f, "yyyy-MM-dd-HH-mm-ss"),
- MINUTELY(MINUTE, 60 * SECONDLY.weight(), 1.35f, "yyyy-MM-dd-HH-mm"),
- HOURLY(HOUR_OF_DAY, 60 * MINUTELY.weight(), 1.3f, "yyyy-MM-dd-HH"),
- DAILY(DAY_OF_MONTH, 24 * HOURLY.weight(), 1f, "yyyy-MM-dd"),
- WEEKLY(WEEK_OF_YEAR, 7 * DAILY.weight(), 0.7f, "yyyy-'W'ww"),
- MONTHLY(MONTH, 30 * DAILY.weight(), 0.6f, "yyyy-MM"),
- QUARTERLY(MONTH, 3 * MONTHLY.weight(), 0.55f, "yyyy-MM"),
- YEARLY(YEAR, 12 * MONTHLY.weight(), 0.52f, "yyyy"),
- CONTINUOUS(Calendar.SECOND, 1, 1.5f, "yyyy-MM-dd-HH-mm-ss");
+ SECONDLY("second", SECOND, 1000, 1.4f, "yyyy-MM-dd-HH-mm-ss"),
+ MINUTELY("minute", MINUTE, 60 * SECONDLY.weight(), 1.35f, "yyyy-MM-dd-HH-mm"),
+ HOURLY("hour", HOUR_OF_DAY, 60 * MINUTELY.weight(), 1.3f, "yyyy-MM-dd-HH"),
+ DAILY("day", DAY_OF_MONTH, 24 * HOURLY.weight(), 1f, "yyyy-MM-dd"),
+ WEEKLY("week", WEEK_OF_YEAR, 7 * DAILY.weight(), 0.7f, "YYYY-'W'ww"),
+ MONTHLY("month", MONTH, 30 * DAILY.weight(), 0.6f, "yyyy-MM"),
+ QUARTERLY("quarter", MONTH, 3 * MONTHLY.weight(), 0.55f, "yyyy-MM"),
+ YEARLY("year", YEAR, 12 * MONTHLY.weight(), 0.52f, "yyyy"),
+ CONTINUOUS("continuous", Calendar.SECOND, 1, 1.5f, "yyyy-MM-dd-HH-mm-ss");
public static final long MIN_INTERVAL = values()[0].weight();
+ @Getter
+ private String unitName;
private final int calendarField;
private final long weight;
/**
@@ -159,7 +165,8 @@ public enum UpdatePeriod implements Named {
private static ThreadLocal<DateFormat> quarterlyFormat;
private static ThreadLocal<DateFormat> yearlyFormat;
- UpdatePeriod(int calendarField, long diff, float normalizationFactor, String format) {
+ UpdatePeriod(String unitName, int calendarField, long diff, float normalizationFactor, String format) {
+ this.unitName = unitName;
this.calendarField = calendarField;
this.weight = diff;
this.normalizationFactor = normalizationFactor;
@@ -178,6 +185,15 @@ public enum UpdatePeriod implements Named {
return DateUtil.getNumberofDaysInMonth(date) * DAILY.weight();
}
+ public static UpdatePeriod fromUnitName(String unitName) throws LensException {
+ for (UpdatePeriod up : values()) {
+ if (up.getUnitName().equals(unitName)) {
+ return up;
+ }
+ }
+ throw new LensException(LensCubeErrorCode.INVALID_TIME_UNIT.getValue(), unitName);
+ }
+
public DateFormat format() {
switch (this) {
case CONTINUOUS:
@@ -221,21 +237,36 @@ public enum UpdatePeriod implements Named {
}
public Date truncate(Date date) {
- if (this.equals(UpdatePeriod.WEEKLY)) {
+ switch (this) {
+ case WEEKLY:
Date truncDate = DateUtils.truncate(date, Calendar.DAY_OF_MONTH);
Calendar cal = Calendar.getInstance();
cal.setTime(truncDate);
cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
return cal.getTime();
- } else if (this.equals(UpdatePeriod.QUARTERLY)) {
+ case QUARTERLY:
Date dt = DateUtils.truncate(date, this.calendarField());
dt.setMonth(dt.getMonth() - dt.getMonth() % 3);
return dt;
- } else {
+ default:
return DateUtils.truncate(date, this.calendarField());
}
}
+ public Calendar truncate(Calendar calendar) {
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(truncate(calendar.getTime()));
+ return cal;
+ }
+
+ public void increment(Calendar calendar, int increment) {
+ switch (this) {
+ case QUARTERLY:
+ increment *= 3;
+ }
+ calendar.add(calendarField(), increment);
+ }
+
public static class UpdatePeriodComparator implements Comparator<UpdatePeriod> {
@Override
public int compare(UpdatePeriod o1, UpdatePeriod o2) {
http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
index c82da7b..a6cb8da 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
@@ -84,11 +84,10 @@ public class StoreAllPartitionTimeline extends PartitionTimeline {
if (partitionsStr == null) {
return true;
}
- boolean ret = true;
for (String s : StringUtils.split(partitionsStr, ",")) {
- ret &= add(TimePartition.of(getUpdatePeriod(), s));
+ add(TimePartition.of(getUpdatePeriod(), s));
}
- return ret;
+ return true;
}
@Override
http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
index 486c6b3..90ea63c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
@@ -50,7 +50,18 @@ public final class DateUtil {
* NOW -> new java.util.Date() NOW-7DAY -> a date one week earlier NOW (+-)
* <NUM>UNIT or Hardcoded dates in DD-MM-YYYY hh:mm:ss,sss
*/
- public static final String UNIT = "year|month|week|day|hour|minute|second";
+ public static final String UNIT;
+
+ static {
+ StringBuilder sb = new StringBuilder();
+ String sep = "";
+ for (UpdatePeriod up : UpdatePeriod.values()) {
+ sb.append(sep).append(up.getUnitName());
+ sep = "|";
+ }
+ UNIT = sb.toString();
+ }
+
public static final String GRANULARITY = "\\.(" + UNIT + ")";
public static final String RELATIVE = "(now){1}(" + GRANULARITY + "){0,1}";
public static final Pattern P_RELATIVE = Pattern.compile(RELATIVE, Pattern.CASE_INSENSITIVE);
@@ -127,9 +138,11 @@ public final class DateUtil {
return resolveAbsoluteDate(str);
}
}
+
public static String relativeToAbsolute(String relative) throws LensException {
return relativeToAbsolute(relative, new Date());
}
+
public static String relativeToAbsolute(String relative, Date now) throws LensException {
if (RELDATE_VALIDATOR.matcher(relative).matches()) {
return ABSDATE_PARSER.get().format(resolveRelativeDate(relative, now));
@@ -166,24 +179,7 @@ public final class DateUtil {
Matcher granularityMatcher = P_UNIT.matcher(nowWithGranularity);
if (granularityMatcher.find()) {
String unit = granularityMatcher.group().toLowerCase();
- if ("year".equals(unit)) {
- calendar = DateUtils.truncate(calendar, YEAR);
- } else if ("month".equals(unit)) {
- calendar = DateUtils.truncate(calendar, MONTH);
- } else if ("week".equals(unit)) {
- calendar.set(Calendar.DAY_OF_WEEK, Calendar.SUNDAY);
- calendar = DateUtils.truncate(calendar, DAY_OF_MONTH);
- } else if ("day".equals(unit)) {
- calendar = DateUtils.truncate(calendar, DAY_OF_MONTH);
- } else if ("hour".equals(unit)) {
- calendar = DateUtils.truncate(calendar, Calendar.HOUR_OF_DAY);
- } else if ("minute".equals(unit)) {
- calendar = DateUtils.truncate(calendar, Calendar.MINUTE);
- } else if ("second".equals(unit)) {
- calendar = DateUtils.truncate(calendar, Calendar.SECOND);
- } else {
- throw new LensException(LensCubeErrorCode.INVALID_TIME_UNIT.getValue(), unit);
- }
+ calendar = UpdatePeriod.fromUnitName(granularityMatcher.group().toLowerCase()).truncate(calendar);
}
}
@@ -386,11 +382,9 @@ public final class DateUtil {
case CONTINUOUS:
return getMilliSecondCoveringInfo(from, to, 1000);
case MINUTELY:
- return getMilliSecondCoveringInfo(from, to, 1000 * 60);
case HOURLY:
- return getMilliSecondCoveringInfo(from, to, 1000 * 60 * 60);
case DAILY:
- return getMilliSecondCoveringInfo(from, to, 1000 * 60 * 60 * 24);
+ return getMilliSecondCoveringInfo(from, to, interval.weight());
case WEEKLY:
return getWeeklyCoveringInfo(from, to);
case MONTHLY:
@@ -404,7 +398,7 @@ public final class DateUtil {
}
}
- private static CoveringInfo getMilliSecondCoveringInfo(Date from, Date to, int millisInInterval) {
+ private static CoveringInfo getMilliSecondCoveringInfo(Date from, Date to, long millisInInterval) {
long diff = to.getTime() - from.getTime();
return new CoveringInfo((int) (diff / millisInInterval), diff % millisInInterval == 0);
}
@@ -419,7 +413,11 @@ public final class DateUtil {
}
public static int getTimeDiff(Date fromDate, Date toDate, UpdatePeriod updatePeriod) {
- return getCoveringInfo(fromDate, toDate, updatePeriod).getCountBetween();
+ if (fromDate.before(toDate)) {
+ return getCoveringInfo(fromDate, toDate, updatePeriod).getCountBetween();
+ } else {
+ return -getCoveringInfo(toDate, fromDate, updatePeriod).getCountBetween();
+ }
}
@Data
@@ -436,11 +434,11 @@ public final class DateUtil {
@EqualsAndHashCode
static class TimeDiff {
int quantity;
- int calendarField;
+ UpdatePeriod updatePeriod;
- public TimeDiff(int quantity, int calendarField) {
+ private TimeDiff(int quantity, UpdatePeriod updatePeriod) {
this.quantity = quantity;
- this.calendarField = calendarField;
+ this.updatePeriod = updatePeriod;
}
static TimeDiff parseFrom(String diffStr) throws LensException {
@@ -461,34 +459,17 @@ public final class DateUtil {
Matcher unitMatcher = P_UNIT.matcher(diffStr);
if (unitMatcher.find()) {
- String unit = unitMatcher.group().toLowerCase();
- if ("year".equals(unit)) {
- return new TimeDiff(qty, YEAR);
- } else if ("month".equals(unit)) {
- return new TimeDiff(qty, MONTH);
- } else if ("week".equals(unit)) {
- return new TimeDiff(7 * qty, DAY_OF_MONTH);
- } else if ("day".equals(unit)) {
- return new TimeDiff(qty, DAY_OF_MONTH);
- } else if ("hour".equals(unit)) {
- return new TimeDiff(qty, HOUR_OF_DAY);
- } else if ("minute".equals(unit)) {
- return new TimeDiff(qty, MINUTE);
- } else if ("second".equals(unit)) {
- return new TimeDiff(qty, SECOND);
- } else {
- throw new LensException(LensCubeErrorCode.INVALID_TIME_UNIT.getValue(), unit);
- }
+ return new TimeDiff(qty, UpdatePeriod.fromUnitName(unitMatcher.group().toLowerCase()));
}
- return new TimeDiff(0, SECOND);
+ return new TimeDiff(0, UpdatePeriod.CONTINUOUS);
}
public Date offsetFrom(Date time) {
- return DateUtils.add(time, calendarField, quantity);
+ return DateUtils.add(time, updatePeriod.calendarField(), quantity);
}
public Date negativeOffsetFrom(Date time) {
- return DateUtils.add(time, calendarField, -quantity);
+ return DateUtils.add(time, updatePeriod.calendarField(), -quantity);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java
index 5444e71..b02499e 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java
@@ -197,7 +197,7 @@ public class TimeRange {
@Override
public Date next() {
Date cur = calendar.getTime();
- calendar.add(updatePeriod.calendarField(), increment);
+ updatePeriod.increment(calendar, increment);
counter++;
return cur;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index c2980af..7d5ed0f 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -28,6 +28,7 @@ import org.apache.lens.cube.metadata.ReferencedDimAtrribute.ChainRefCol;
import org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline;
import org.apache.lens.cube.metadata.timeline.PartitionTimeline;
import org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline;
+import org.apache.lens.cube.metadata.timeline.TestPartitionTimelines;
import org.apache.lens.cube.parse.TimeRange;
import org.apache.lens.server.api.error.LensException;
@@ -1447,26 +1448,12 @@ public class TestCubeMetastoreClient {
private void assertSameTimelines(String factName, String[] storages, UpdatePeriod updatePeriod, String[] partCols)
throws HiveException, LensException {
for (String partCol : partCols) {
- PartitionTimeline[] timelines = new PartitionTimeline[storages.length];
+ ArrayList<PartitionTimeline> timelines = Lists.newArrayList();
for (int i = 0; i < storages.length; i++) {
- timelines[i] = client.partitionTimelineCache.get(factName, storages[i], updatePeriod, partCol);
+ timelines.add(client.partitionTimelineCache.get(factName, storages[i], updatePeriod, partCol));
}
- for (int i = 0; i < timelines.length; i++) {
- for (int j = i + 1; j < timelines.length; j++) {
- assertSameTimelines(timelines[i], timelines[j]);
- }
- }
- }
- }
-
- private void assertSameTimelines(PartitionTimeline timeline1, PartitionTimeline timeline2) {
- Iterator<TimePartition> iter1 = timeline1.iterator();
- Iterator<TimePartition> iter2 = timeline2.iterator();
- while (iter1.hasNext()) {
- Assert.assertTrue(iter2.hasNext());
- assertEquals(iter1.next(), iter2.next());
+ TestPartitionTimelines.assertSameTimelines(timelines);
}
- Assert.assertFalse(iter2.hasNext());
}
private StoragePartitionDesc getStoragePartSpec(String cubeFactName, UpdatePeriod updatePeriod,
http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java
index 334ea4f..2cbbd0c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java
@@ -25,44 +25,46 @@ import java.util.Date;
import org.apache.lens.server.api.error.LensException;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
public class TestTimePartition {
public static final Date NOW = new Date();
- @Test
- public void test() throws LensException {
- // Test for all update periods
- for (UpdatePeriod up : UpdatePeriod.values()) {
- // Normal date object parsable
- String nowStr = up.format().format(NOW);
+ @DataProvider(name = "update-periods")
+ public Object[][] provideUpdatePeriods() {
+ return UpdatePeriodTest.provideUpdatePeriods();
+ }
- // Create partition by date object or it's string representation -- both should be same.
- TimePartition nowPartition = TimePartition.of(up, NOW);
- TimePartition nowStrPartition = TimePartition.of(up, nowStr);
- assertEquals(nowPartition, nowStrPartition);
-
- // Test next and previous
- assertTrue(nowPartition.next().after(nowPartition));
- assertTrue(nowPartition.previous().before(nowPartition));
-
- // date parse failures should give lens exception
- assertEquals(getLensExceptionFromPartitionParsing(up, "garbage").getMessage(),
- TimePartition.getWrongUpdatePeriodMessage(up, "garbage"));
- getLensExceptionFromPartitionParsing(up, (Date) null);
- getLensExceptionFromPartitionParsing(up, (String) null);
- getLensExceptionFromPartitionParsing(up, "");
-
- // parse with other update periods
- for (UpdatePeriod up2 : UpdatePeriod.values()) {
- // handles the equality case and the case where monthly-quarterly have same format strings.
- if (up.formatStr().equals(up2.formatStr())) {
- continue;
- }
- // Parsing a string representation with differnet update period should give lens exception.
- assertEquals(getLensExceptionFromPartitionParsing(up2, nowStr).getMessage(),
- TimePartition.getWrongUpdatePeriodMessage(up2, nowStr));
+ @Test(dataProvider = "update-periods")
+ public void test(UpdatePeriod up) throws LensException {
+ // Normal date object parsable
+ String nowStr = up.format().format(NOW);
+ // Create partition by date object or it's string representation -- both should be same.
+ TimePartition nowPartition = TimePartition.of(up, NOW);
+ TimePartition nowStrPartition = TimePartition.of(up, nowStr);
+ assertEquals(nowPartition, nowStrPartition);
+
+ // Test next and previous
+ assertTrue(nowPartition.next().after(nowPartition));
+ assertTrue(nowPartition.previous().before(nowPartition));
+
+ // date parse failures should give lens exception
+ assertEquals(getLensExceptionFromPartitionParsing(up, "garbage").getMessage(),
+ TimePartition.getWrongUpdatePeriodMessage(up, "garbage"));
+ getLensExceptionFromPartitionParsing(up, (Date) null);
+ getLensExceptionFromPartitionParsing(up, (String) null);
+ getLensExceptionFromPartitionParsing(up, "");
+
+ // parse with other update periods
+ for (UpdatePeriod up2 : UpdatePeriod.values()) {
+ // handles the equality case and the case where monthly-quarterly have same format strings.
+ if (up.formatStr().equals(up2.formatStr())) {
+ continue;
}
+ // Parsing a string representation with differnet update period should give lens exception.
+ assertEquals(getLensExceptionFromPartitionParsing(up2, nowStr).getMessage(),
+ TimePartition.getWrongUpdatePeriodMessage(up2, nowStr));
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/test/java/org/apache/lens/cube/metadata/UpdatePeriodTest.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/UpdatePeriodTest.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/UpdatePeriodTest.java
new file mode 100644
index 0000000..b21341d
--- /dev/null
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/UpdatePeriodTest.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import static org.testng.Assert.assertEquals;
+
+import java.util.Date;
+import java.util.Random;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+
+public class UpdatePeriodTest {
+ public static class RandomDateGenerator {
+ Random random = new Random();
+ long begin = 1377424977000L;
+ long end = 1598349777000L;
+
+ public Date nextDate() {
+ return new Date(begin + (random.nextLong() % (end - begin)));
+ }
+ }
+
+ @DataProvider(name = "update-periods")
+ public static Object[][] provideUpdatePeriods() {
+ UpdatePeriod[] values = UpdatePeriod.values();
+ Object[][] ret = new Object[values.length][1];
+ for (int i = 0; i < values.length; i++) {
+ ret[i][0] = values[i];
+ }
+ return ret;
+ }
+
+ @Test(dataProvider = "update-periods")
+ public void testFormat(UpdatePeriod period) throws Exception {
+ RandomDateGenerator randomDateGenerator = new RandomDateGenerator();
+ for (int i = 0; i < 5000; i++) {
+ Date randomDate = randomDateGenerator.nextDate();
+ randomDate = period.truncate(randomDate);
+ assertEquals(randomDate, period.format().parse(period.format().format(randomDate)));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
index 0268bbf..3f9e125 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
@@ -18,25 +18,28 @@
*/
package org.apache.lens.cube.metadata.timeline;
+import static org.testng.Assert.*;
+
+import java.lang.reflect.InvocationTargetException;
import java.util.*;
import org.apache.lens.cube.metadata.TestTimePartition;
import org.apache.lens.cube.metadata.TimePartition;
import org.apache.lens.cube.metadata.UpdatePeriod;
+import org.apache.lens.cube.metadata.UpdatePeriodTest;
import org.apache.lens.server.api.error.LensException;
-import org.testng.Assert;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import com.beust.jcommander.internal.Lists;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class TestPartitionTimelines {
private static final String TABLE_NAME = "storage_fact";
- public static final UpdatePeriod PERIOD = UpdatePeriod.HOURLY;
private static final String PART_COL = "pt";
private static final List<Class<? extends PartitionTimeline>> TIMELINE_IMPLEMENTATIONS = Arrays.asList(
StoreAllPartitionTimeline.class,
@@ -44,46 +47,48 @@ public class TestPartitionTimelines {
RangesPartitionTimeline.class
);
- @Test
- public void testPropertiesContractsForAllSubclasses() throws LensException {
- for (Class<? extends PartitionTimeline> clazz : TIMELINE_IMPLEMENTATIONS) {
- testPropertiesContract(clazz);
+ @DataProvider(name = "update-periods")
+ public Object[][] provideUpdatePeriods() {
+ return UpdatePeriodTest.provideUpdatePeriods();
+ }
+
+ @DataProvider(name = "update-periods-and-timeline-classes")
+ public Object[][] provideUpdatePeriodsAndTimelineClasses() {
+ UpdatePeriod[] values = UpdatePeriod.values();
+ Object[][] ret = new Object[values.length * TIMELINE_IMPLEMENTATIONS.size()][2];
+ for (int i = 0; i < values.length; i++) {
+ for (int j = 0; j < TIMELINE_IMPLEMENTATIONS.size(); j++) {
+ ret[TIMELINE_IMPLEMENTATIONS.size() * i + j] = new Object[]{
+ values[i],
+ TIMELINE_IMPLEMENTATIONS.get(j),
+ };
+ }
}
+ return ret;
}
- @Test
- public void testEquivalence() throws LensException {
+
+ @Test(dataProvider = "update-periods")
+ public void testEquivalence(UpdatePeriod period) throws LensException, InvocationTargetException,
+ NoSuchMethodException, InstantiationException, IllegalAccessException {
+ final Random randomGenerator = new Random();
for (int j = 0; j < 10; j++) {
- Random randomGenerator = new Random();
List<PartitionTimeline> timelines = Lists.newArrayList();
for (Class<? extends PartitionTimeline> clazz : TIMELINE_IMPLEMENTATIONS) {
- timelines.add(getInstance(clazz));
+ timelines.add(getInstance(clazz, period));
}
final List<TimePartition> addedPartitions = Lists.newArrayList();
- for (int i = 0; i < 200; i++) {
- int randomInt = randomGenerator.nextInt(100) - 50;
- TimePartition part = TimePartition.of(PERIOD, TestTimePartition.timeAtDiff(TestTimePartition.NOW, PERIOD,
+ for (int i = 0; i < 20; i++) {
+ int randomInt = randomGenerator.nextInt(10) - 5;
+ TimePartition part = TimePartition.of(period, TestTimePartition.timeAtDiff(TestTimePartition.NOW, period,
randomInt));
addedPartitions.add(part);
for (PartitionTimeline timeline : timelines) {
timeline.add(part);
}
+ assertSameTimelines(timelines);
}
- Iterator<TimePartition> sourceOfTruth = timelines.get(0).iterator();
- List<Iterator<TimePartition>> otherIterators = Lists.newArrayList();
- for (int i = 1; i < TIMELINE_IMPLEMENTATIONS.size() - 1; i++) {
- otherIterators.add(timelines.get(i).iterator());
- }
- while (sourceOfTruth.hasNext()) {
- TimePartition cur = sourceOfTruth.next();
- for (Iterator<TimePartition> iterator : otherIterators) {
- Assert.assertTrue(iterator.hasNext());
- Assert.assertEquals(iterator.next(), cur);
- }
- }
- for (Iterator<TimePartition> iterator : otherIterators) {
- Assert.assertFalse(iterator.hasNext());
- }
+ assertSameTimelines(timelines);
Collections.shuffle(addedPartitions);
Iterator<TimePartition> iter = addedPartitions.iterator();
while (iter.hasNext()) {
@@ -92,62 +97,80 @@ public class TestPartitionTimelines {
if (!addedPartitions.contains(part)) {
for (PartitionTimeline timeline : timelines) {
timeline.drop(part);
+ assertTrue(timeline.isConsistent());
}
}
}
for (PartitionTimeline timeline : timelines) {
- Assert.assertTrue(timeline.isEmpty());
+ assertTrue(timeline.isEmpty());
}
}
}
+ public static void assertSameTimelines(List<PartitionTimeline> timelines) {
+ List<Iterator<TimePartition>> iterators = Lists.newArrayList();
+ for (PartitionTimeline timeline : timelines) {
+ iterators.add(timeline.iterator());
+ }
- private <T extends PartitionTimeline> T getInstance(Class<T> clz) {
- try {
- return clz.getConstructor(String.class, UpdatePeriod.class, String.class)
- .newInstance(TABLE_NAME, PERIOD, PART_COL);
- } catch (Exception e) {
- log.error("Error while getting instance.", e);
+ while (iterators.get(0).hasNext()) {
+ Map<Class, TimePartition> parts = Maps.newHashMap();
+ for (Iterator<TimePartition> iterator : iterators) {
+ assertTrue(iterator.hasNext());
+ parts.put(iterator.getClass(), iterator.next());
+ }
+ assertEquals(new HashSet<>(parts.values()).size(), 1, "More than one values for next: " + parts.values());
}
- return null;
+ for (Iterator<TimePartition> iterator : iterators) {
+ assertFalse(iterator.hasNext());
+ }
+ }
+
+
+ private <T extends PartitionTimeline> T getInstance(Class<T> clz, UpdatePeriod period) throws
+ NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
+ return clz.getConstructor(String.class, UpdatePeriod.class, String.class)
+ .newInstance(TABLE_NAME, period, PART_COL);
}
- private <T extends PartitionTimeline> void testPropertiesContract(Class<T> clz) throws LensException {
+ @Test(dataProvider = "update-periods-and-timeline-classes")
+ public <T extends PartitionTimeline> void testPropertiesContract(UpdatePeriod period, Class<T> clz) throws
+ LensException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
// Make two instances, one to modify, other to validate against
- T inst1 = getInstance(clz);
- T inst2 = getInstance(clz);
+ T inst1 = getInstance(clz, period);
+ T inst2 = getInstance(clz, period);
// whenever we'll init from props, timeline should become empty.
Map<String, String> props = inst1.toProperties();
- Assert.assertTrue(inst2.initFromProperties(props));
+ assertTrue(inst2.initFromProperties(props));
// init from props of an empty timeline: should succeed and make the timeline empty
- Assert.assertEquals(inst1, inst2);
- Assert.assertTrue(inst1.isEmpty());
- Assert.assertTrue(inst2.isEmpty());
+ assertEquals(inst1, inst2);
+ assertTrue(inst1.isEmpty());
+ assertTrue(inst2.isEmpty());
// Add single partition and test for non-equivalence
- Assert.assertTrue(inst1.add(TimePartition.of(PERIOD, TestTimePartition.NOW)));
- Assert.assertFalse(inst1.equals(inst2));
+ assertTrue(inst1.add(TimePartition.of(period, TestTimePartition.NOW)));
+ assertFalse(inst1.equals(inst2));
// add same parittion in other timeline, test for equality
- Assert.assertTrue(inst2.add(TimePartition.of(PERIOD, TestTimePartition.NOW)));
- Assert.assertTrue(inst1.isConsistent());
- Assert.assertTrue(inst2.isConsistent());
- Assert.assertEquals(inst1, inst2);
+ assertTrue(inst2.add(TimePartition.of(period, TestTimePartition.NOW)));
+ assertTrue(inst1.isConsistent());
+ assertTrue(inst2.isConsistent());
+ assertEquals(inst1, inst2);
// init with blank properties. Should become empty
- Assert.assertTrue(inst2.initFromProperties(props));
- Assert.assertFalse(inst1.equals(inst2));
+ assertTrue(inst2.initFromProperties(props));
+ assertFalse(inst1.equals(inst2));
// init from properties of timeline with single partition.
- Assert.assertTrue(inst2.initFromProperties(inst1.toProperties()));
- Assert.assertEquals(inst1, inst2);
+ assertTrue(inst2.initFromProperties(inst1.toProperties()));
+ assertEquals(inst1, inst2);
// clear timelines
inst1.initFromProperties(props);
inst2.initFromProperties(props);
// Make sparse partition range in one, init other from its properties. Test equality.
- for (int i = 0; i < 5000; i++) {
- Assert.assertTrue(inst1.add(TimePartition.of(PERIOD, TestTimePartition.timeAtDiff(TestTimePartition.NOW, PERIOD,
+ for (int i = 0; i < 500; i++) {
+ assertTrue(inst1.add(TimePartition.of(period, TestTimePartition.timeAtDiff(TestTimePartition.NOW, period,
i * 2))));
}
- Assert.assertTrue(inst1.isConsistent());
+ assertTrue(inst1.isConsistent());
inst2.initFromProperties(inst1.toProperties());
- Assert.assertTrue(inst2.isConsistent());
- Assert.assertEquals(inst1, inst2);
+ assertTrue(inst2.isConsistent());
+ assertEquals(inst1, inst2);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java
index 89d210f..ab88fbe 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java
@@ -262,7 +262,7 @@ public class TestDateUtil {
assertEquals(diffs.size(), 1);
TimeDiff minusFourDaysDiff = diffs.iterator().next();
assertEquals(minusFourDaysDiff.quantity, -4);
- assertEquals(minusFourDaysDiff.calendarField, DAY_OF_MONTH);
+ assertEquals(minusFourDaysDiff.updatePeriod, DAILY);
diffs.clear();
for (String diffStr : plusFourDays) {
@@ -271,7 +271,7 @@ public class TestDateUtil {
assertEquals(diffs.size(), 1);
TimeDiff plusFourDaysDiff = diffs.iterator().next();
assertEquals(plusFourDaysDiff.quantity, 4);
- assertEquals(plusFourDaysDiff.calendarField, DAY_OF_MONTH);
+ assertEquals(plusFourDaysDiff.updatePeriod, DAILY);
Date now = new Date();
assertEquals(minusFourDaysDiff.offsetFrom(plusFourDaysDiff.offsetFrom(now)), now);
assertEquals(plusFourDaysDiff.offsetFrom(minusFourDaysDiff.offsetFrom(now)), now);