You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/04/15 21:50:07 UTC
[33/50] [abbrv] incubator-lens git commit: LENS-462 : Handle the case
when a table property becomes large for a timeline (Rajat Khandelwal via
amareshwari)
LENS-462 : Handle the case when a table property becomes large for a timeline (Rajat Khandelwal via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/21102e6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/21102e6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/21102e6b
Branch: refs/heads/current-release-line
Commit: 21102e6b96c3139563ef02449a6857bf48d983c4
Parents: 07492f1
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Tue Apr 7 17:50:48 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Apr 7 17:50:48 2015 +0530
----------------------------------------------------------------------
.../lens/cube/metadata/CubeMetastoreClient.java | 95 ++++++++------
.../lens/cube/metadata/MetastoreUtil.java | 16 ++-
.../lens/cube/metadata/TimePartition.java | 77 ++---------
.../lens/cube/metadata/TimePartitionRange.java | 130 +++++++++++++++++++
.../timeline/EndsAndHolesPartitionTimeline.java | 9 +-
.../timeline/RangesPartitionTimeline.java | 27 ++--
.../timeline/StoreAllPartitionTimeline.java | 5 +-
.../lens/cube/parse/StorageTableResolver.java | 2 +-
.../lens/cube/metadata/TestTimePartition.java | 94 ++++++++++++--
.../timeline/TestPartitionTimelines.java | 38 ++++--
.../apache/lens/cube/parse/CubeTestSetup.java | 6 +-
11 files changed, 335 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/21102e6b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 1835d2f..11ef7ec 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -186,56 +186,72 @@ public class CubeMetastoreClient {
if (get(storageTableName) == null) {
synchronized (this) {
if (get(storageTableName) == null) {
- log.info("loading timeline from all partitions for storage table: " + storageTableName);
- // not found in memory, try loading from table properties.
Table storageTable = getTable(storageTableName);
- if (!"true".equalsIgnoreCase(
- storageTable.getParameters().get(MetastoreUtil.getPartitoinTimelineCachePresenceKey()))) {
- // Not found in table properties either, compute from all partitions of the fact-storage table.
- // First make sure all combinations of update period and partition column have an entry even
- // if no partitions exist
- log.info("loading from all partitions");
- if (getCubeFact(fact).getUpdatePeriods() != null && getCubeFact(fact).getUpdatePeriods().get(
- storage) != null) {
- for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
- for (String partCol : getTimePartsOfTable(storageTable)) {
- partitionTimelineCache.ensureEntry(storageTableName, updatePeriod, partCol);
- }
- }
- }
- // Then add all existing partitions for batch addition in respective timelines.
- List<String> timeParts = getTimePartsOfTable(storageTable);
- List<FieldSchema> partCols = storageTable.getPartCols();
- for (Partition partition : getPartitionsByFilter(storageTableName, null)) {
- UpdatePeriod period = deduceUpdatePeriod(partition);
- List<String> values = partition.getValues();
- for (int i = 0; i < partCols.size(); i++) {
- if (timeParts.contains(partCols.get(i).getName())) {
- partitionTimelineCache.addForBatchAddition(storageTableName, period, partCols.get(i).getName(),
- values.get(i));
- }
- }
+ if ("true".equalsIgnoreCase(storageTable.getParameters().get(
+ MetastoreUtil.getPartitoinTimelineCachePresenceKey()))) {
+ try {
+ loadTimelinesFromTableProperties(fact, storage);
+ } catch (Exception e) {
+ // Ideally this should never come. But since we have another source,
+ // let's piggyback on that for loading timeline
+ log.error("Error while loading timelines from table properties.", e);
+ loadTimelinesFromAllPartitions(fact, storage);
}
- // commit all batch addition for the storage table,
- // which will in-turn commit all batch additions in all it's timelines.
- commitAllBatchAdditions(storageTableName);
} else {
- // found in table properties, load from there.
- log.info("loading from table properties");
- for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
- for (String partCol : getTimePartsOfTable(storageTableName)) {
- ensureEntry(storageTableName, updatePeriod, partCol).init(storageTable);
- }
- }
+ loadTimelinesFromAllPartitions(fact, storage);
}
}
}
+ log.info("timeline for " + storageTableName + " is: " + get(storageTableName));
}
// return the final value from memory
return get(storageTableName);
// RESUME CHECKSTYLE CHECK DoubleCheckedLockingCheck
}
+ private void loadTimelinesFromAllPartitions(String fact, String storage) throws HiveException, LensException {
+ // Not found in table properties either, compute from all partitions of the fact-storage table.
+ // First make sure all combinations of update period and partition column have an entry even
+ // if no partitions exist
+ String storageTableName = MetastoreUtil.getStorageTableName(fact, Storage.getPrefix(storage));
+ log.info("loading from all partitions: " + storageTableName);
+ Table storageTable = getTable(storageTableName);
+ if (getCubeFact(fact).getUpdatePeriods() != null && getCubeFact(fact).getUpdatePeriods().get(
+ storage) != null) {
+ for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
+ for (String partCol : getTimePartsOfTable(storageTable)) {
+ ensureEntry(storageTableName, updatePeriod, partCol);
+ }
+ }
+ }
+ // Then add all existing partitions for batch addition in respective timelines.
+ List<String> timeParts = getTimePartsOfTable(storageTable);
+ List<FieldSchema> partCols = storageTable.getPartCols();
+ for (Partition partition : getPartitionsByFilter(storageTableName, null)) {
+ UpdatePeriod period = deduceUpdatePeriod(partition);
+ List<String> values = partition.getValues();
+ for (int i = 0; i < partCols.size(); i++) {
+ if (timeParts.contains(partCols.get(i).getName())) {
+ addForBatchAddition(storageTableName, period, partCols.get(i).getName(), values.get(i));
+ }
+ }
+ }
+ // commit all batch addition for the storage table,
+ // which will in-turn commit all batch additions in all it's timelines.
+ commitAllBatchAdditions(storageTableName);
+ }
+
+ private void loadTimelinesFromTableProperties(String fact, String storage) throws HiveException, LensException {
+ // found in table properties, load from there.
+ String storageTableName = MetastoreUtil.getStorageTableName(fact, Storage.getPrefix(storage));
+ log.info("loading from table properties: " + storageTableName);
+ for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
+ for (String partCol : getTimePartsOfTable(storageTableName)) {
+ ensureEntry(storageTableName, updatePeriod, partCol).init(getTable(storageTableName));
+ }
+ }
+ }
+
/**
* Adds given partition(for storageTable, updatePeriod, partitionColum=partition) for batch addition in an
* appropriate timeline object. Ignore if partition is not valid.
@@ -277,8 +293,7 @@ public class CubeMetastoreClient {
get(storageTable).get(updatePeriod).put(partitionColumn, PartitionTimelineFactory.get(
CubeMetastoreClient.this, storageTable, updatePeriod, partitionColumn));
}
- PartitionTimeline ret = get(storageTable).get(updatePeriod).get(partitionColumn);
- return ret;
+ return get(storageTable).get(updatePeriod).get(partitionColumn);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/21102e6b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index 0be8e5f..203ff58 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -402,12 +402,18 @@ public class MetastoreUtil {
}
public static String getNamedStringValue(Map<String, String> props, String key) {
- int size = Integer.parseInt(props.get(key + ".size"));
- StringBuilder valueStr = new StringBuilder();
- for (int i = 0; i < size; i++) {
- valueStr.append(props.get(key + i));
+ if (props.containsKey(key + ".size")) {
+ int size = Integer.parseInt(props.get(key + ".size"));
+ StringBuilder valueStr = new StringBuilder();
+ for (int i = 0; i < size; i++) {
+ valueStr.append(props.get(key + i));
+ }
+ return valueStr.toString();
+ } else if (props.containsKey(key)) {
+ return props.get(key);
+ } else {
+ return null;
}
- return valueStr.toString();
}
public static String getObjectStr(Collection<?> set) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/21102e6b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
index b948467..8f088c0 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
@@ -21,7 +21,6 @@ package org.apache.lens.cube.metadata;
import java.text.ParseException;
import java.util.Calendar;
import java.util.Date;
-import java.util.Iterator;
import org.apache.lens.api.LensException;
@@ -32,7 +31,7 @@ import lombok.NonNull;
/** stores a partition's update period, date and string representation. Provides some utility methods around it */
@Data
-public class TimePartition implements Comparable<TimePartition> {
+public class TimePartition implements Comparable<TimePartition>, Named {
private static final String UPDATE_PERIOD_WRONG_ERROR_MESSAGE = "Update period %s not correct for parsing %s";
private final UpdatePeriod updatePeriod;
private final Date date;
@@ -89,6 +88,9 @@ public class TimePartition implements Comparable<TimePartition> {
public TimePartition partitionAtDiff(int increment) {
Calendar cal = Calendar.getInstance();
cal.setTime(date);
+ if (getUpdatePeriod().equals(UpdatePeriod.QUARTERLY)) {
+ increment *= 3;
+ }
cal.add(updatePeriod.calendarField(), increment);
return new TimePartition(updatePeriod, cal.getTime());
}
@@ -125,78 +127,21 @@ public class TimePartition implements Comparable<TimePartition> {
return String.format(UPDATE_PERIOD_WRONG_ERROR_MESSAGE, up, dateString);
}
- public TimePartitionRange rangeUpto(TimePartition to) {
+ public TimePartitionRange rangeUpto(TimePartition to) throws LensException {
return new TimePartitionRange(this, to);
}
- public TimePartitionRange rangeFrom(TimePartition from) {
+ public TimePartitionRange rangeFrom(TimePartition from) throws LensException {
return new TimePartitionRange(from, this);
}
- public TimePartitionRange singletonRange() {
+ public TimePartitionRange singletonRange() throws LensException {
return rangeUpto(next());
}
- /**
- * Range of time partition. [begin,end). i.e. inclusive begin and exclusive end.
- */
- @Data
- public static class TimePartitionRange implements Iterable<TimePartition> {
- private TimePartition begin;
- private TimePartition end;
-
- public TimePartitionRange(TimePartition from, TimePartition to) {
- this.begin = from;
- this.end = to;
- }
-
- @Override
- public String toString() {
- return "[" + begin.getDateString() + ", " + end.getDateString() + ")";
- }
-
- /**
- * returns TimePartition objects starting from begin and upto(excluding) end. interval of iteration is the update
- * period of the partitions. Assumes both partitions have same update period.
- */
- @Override
- public Iterator<TimePartition> iterator() {
-
- return new Iterator<TimePartition>() {
- TimePartition current = begin;
-
- @Override
- public boolean hasNext() {
- return current.before(end);
- }
-
- @Override
- public TimePartition next() {
- TimePartition ret = current;
- current = current.next();
- return ret;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException("remove not supported");
- }
- };
- }
-
- /**
- * @param partition
- * @return begin <= partition < end
- */
- public boolean contains(TimePartition partition) {
- return !partition.before(begin) && partition.before(end);
- }
-
- /**
- * @return if range is empty range.
- */
- public boolean isEmpty() {
- return begin.equals(end);
- }
+ @Override
+ public String getName() {
+ return getDateString();
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/21102e6b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java
new file mode 100644
index 0000000..86c3453
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import java.util.Iterator;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.cube.parse.DateUtil;
+
+import lombok.Data;
+
+/**
+ * Range of time partition. [begin,end). i.e. inclusive begin and exclusive end.
+ */
+@Data
+public class TimePartitionRange implements Iterable<TimePartition>, Named {
+ private TimePartition begin;
+ private TimePartition end;
+
+ public TimePartitionRange(TimePartition begin, TimePartition end) throws LensException {
+ if (end.before(begin)) {
+ throw new LensException("condition of creation of timepartition failed: end>=begin");
+ }
+ if (end.getUpdatePeriod() != begin.getUpdatePeriod()) {
+ throw new LensException("update periods are not same");
+ }
+ this.begin = begin;
+ this.end = end;
+ }
+
+ @Override
+ public String toString() {
+ return "[" + begin.getDateString() + ", " + end.getDateString() + ")";
+ }
+
+ /**
+ * returns TimePartition objects starting from begin and upto(excluding) end. interval of iteration is the update
+ * period of the partitions. Assumes both partitions have same update period.
+ */
+ @Override
+ public Iterator<TimePartition> iterator() {
+
+ return new Iterator<TimePartition>() {
+ TimePartition current = begin;
+
+ @Override
+ public boolean hasNext() {
+ return current.before(end);
+ }
+
+ @Override
+ public TimePartition next() {
+ TimePartition ret = current;
+ current = current.next();
+ return ret;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("remove not supported");
+ }
+ };
+ }
+
+ /**
+ * @param partition
+ * @return begin <= partition < end
+ */
+ public boolean contains(TimePartition partition) {
+ return !partition.before(begin) && partition.before(end);
+ }
+
+ /**
+ * @return if range is empty range.
+ */
+ public boolean isEmpty() {
+ return begin.equals(end);
+ }
+
+ @Override
+ public String getName() {
+ return toString();
+ }
+
+ public static TimePartitionRange parseFrom(UpdatePeriod updatePeriod, String from, String to) throws LensException {
+ boolean incrementFrom = false;
+ boolean incrementTo = false;
+ if (from.charAt(0) == '[') {
+ from = from.substring(1);
+ } else if (from.charAt(0) == '(') {
+ from = from.substring(1);
+ incrementFrom = true;
+ }
+ if (to.charAt(to.length() - 1) == ']') {
+ to = to.substring(0, to.length() - 1);
+ incrementTo = true;
+ } else if (to.charAt(to.length() - 1) == ')') {
+ to = to.substring(0, to.length() - 1);
+ }
+ TimePartition fromPartition = TimePartition.of(updatePeriod, from);
+ TimePartition toPartition = TimePartition.of(updatePeriod, to);
+ if (incrementFrom) {
+ fromPartition = fromPartition.next();
+ }
+ if (incrementTo) {
+ toPartition = toPartition.next();
+ }
+ return new TimePartitionRange(fromPartition, toPartition);
+ }
+
+ public long size() {
+ return DateUtil.getTimeDiff(begin.getDate(), end.getDate(), begin.getUpdatePeriod());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/21102e6b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
index 79e8a62..5867587 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
@@ -22,12 +22,11 @@ package org.apache.lens.cube.metadata.timeline;
import java.util.*;
import org.apache.lens.api.LensException;
+import org.apache.lens.cube.metadata.MetastoreUtil;
import org.apache.lens.cube.metadata.TimePartition;
import org.apache.lens.cube.metadata.UpdatePeriod;
import org.apache.lens.cube.parse.TimeRange;
-import org.apache.commons.lang.StringUtils;
-
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -99,12 +98,12 @@ public class EndsAndHolesPartitionTimeline extends PartitionTimeline {
@Override
public Map<String, String> toProperties() {
HashMap<String, String> ret = Maps.newHashMap();
+ MetastoreUtil.addNameStrings(ret, "holes", holes);
if (isEmpty()) {
return ret;
}
ret.put("first", first.getDateString());
ret.put("latest", latest.getDateString());
- ret.put("holes", StringUtils.join(holes, ","));
return ret;
}
@@ -115,7 +114,7 @@ public class EndsAndHolesPartitionTimeline extends PartitionTimeline {
holes.clear();
String firstStr = properties.get("first");
String latestStr = properties.get("latest");
- String holesStr = properties.get("holes");
+ String holesStr = MetastoreUtil.getNamedStringValue(properties, "holes");
if (!Strings.isNullOrEmpty(firstStr)) {
first = TimePartition.of(getUpdatePeriod(), firstStr);
}
@@ -124,7 +123,7 @@ public class EndsAndHolesPartitionTimeline extends PartitionTimeline {
}
holes = Sets.newTreeSet();
if (!Strings.isNullOrEmpty(holesStr)) {
- for (String hole : properties.get("holes").split("\\s*,\\s*")) {
+ for (String hole : holesStr.split("\\s*,\\s*")) {
holes.add(TimePartition.of(getUpdatePeriod(), hole));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/21102e6b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java
index fb2d0a8..8d80f0b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java
@@ -25,7 +25,9 @@ import java.util.List;
import java.util.Map;
import org.apache.lens.api.LensException;
+import org.apache.lens.cube.metadata.MetastoreUtil;
import org.apache.lens.cube.metadata.TimePartition;
+import org.apache.lens.cube.metadata.TimePartitionRange;
import org.apache.lens.cube.metadata.UpdatePeriod;
import com.google.common.base.Strings;
@@ -42,7 +44,7 @@ import lombok.ToString;
@Data
@ToString(callSuper = true)
public class RangesPartitionTimeline extends PartitionTimeline {
- private List<TimePartition.TimePartitionRange> ranges = Lists.newArrayList();
+ private List<TimePartitionRange> ranges = Lists.newArrayList();
public RangesPartitionTimeline(String storageTableName, UpdatePeriod updatePeriod,
String partCol) {
@@ -103,7 +105,7 @@ public class RangesPartitionTimeline extends PartitionTimeline {
private void mergeRanges() {
for (int i = 0; i < ranges.size() - 1; i++) {
if (ranges.get(i).getEnd().equals(ranges.get(i + 1).getBegin())) {
- TimePartition.TimePartitionRange removed = ranges.remove(i + 1);
+ TimePartitionRange removed = ranges.remove(i + 1);
ranges.get(i).setEnd(removed.getEnd());
i--; // check again at same index
}
@@ -143,32 +145,21 @@ public class RangesPartitionTimeline extends PartitionTimeline {
@Override
public Map<String, String> toProperties() {
HashMap<String, String> ret = Maps.newHashMap();
- if (isEmpty()) {
- return ret;
- }
- StringBuilder sb = new StringBuilder();
- String sep = "";
- for (TimePartition.TimePartitionRange range : ranges) {
- sb.append(sep);
- sep = ",";
- sb.append(range.getBegin()).append(sep).append(range.getEnd());
- }
- ret.put("ranges", sb.toString());
+ MetastoreUtil.addNameStrings(ret, "ranges", ranges);
return ret;
}
@Override
public boolean initFromProperties(Map<String, String> properties) throws LensException {
ranges.clear();
- String rangesStr = properties.get("ranges");
+ String rangesStr = MetastoreUtil.getNamedStringValue(properties, "ranges");
if (!Strings.isNullOrEmpty(rangesStr)) {
String[] split = rangesStr.split("\\s*,\\s*");
if (split.length % 2 == 1) {
throw new LensException("Ranges incomplete");
}
for (int i = 0; i < split.length; i += 2) {
- ranges.add(TimePartition.of(getUpdatePeriod(), split[i]).rangeUpto(TimePartition.of(getUpdatePeriod(),
- split[i + 1])));
+ ranges.add(TimePartitionRange.parseFrom(getUpdatePeriod(), split[i], split[i + 1]));
}
}
return isConsistent();
@@ -203,7 +194,7 @@ public class RangesPartitionTimeline extends PartitionTimeline {
if (isEmpty()) {
return false;
}
- for (TimePartition.TimePartitionRange range : ranges) {
+ for (TimePartitionRange range : ranges) {
if (range.contains(toCheck)) {
return true;
}
@@ -215,7 +206,7 @@ public class RangesPartitionTimeline extends PartitionTimeline {
public Iterator<TimePartition> iterator() {
return new Iterator<TimePartition>() {
- Iterator<TimePartition.TimePartitionRange> uber = ranges.iterator();
+ Iterator<TimePartitionRange> uber = ranges.iterator();
Iterator<TimePartition> cur = null;
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/21102e6b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
index d6ee0a1..2364400 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
@@ -21,6 +21,7 @@ package org.apache.lens.cube.metadata.timeline;
import java.util.*;
import org.apache.lens.api.LensException;
+import org.apache.lens.cube.metadata.MetastoreUtil;
import org.apache.lens.cube.metadata.TimePartition;
import org.apache.lens.cube.metadata.UpdatePeriod;
@@ -70,14 +71,14 @@ public class StoreAllPartitionTimeline extends PartitionTimeline {
@Override
public Map<String, String> toProperties() {
HashMap<String, String> map = Maps.newHashMap();
- map.put("partitions", StringUtils.join(allPartitions, ","));
+ MetastoreUtil.addNameStrings(map, "partitions", allPartitions);
return map;
}
@Override
public boolean initFromProperties(Map<String, String> properties) throws LensException {
allPartitions.clear();
- String partitionsStr = properties.get("partitions");
+ String partitionsStr = MetastoreUtil.getNamedStringValue(properties, "partitions");
if (partitionsStr == null) {
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/21102e6b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 96ca82c..ab2c3f9 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -334,7 +334,7 @@ class StorageTableResolver implements ContextRewriter {
Set<String> nonExistingParts = Sets.newHashSet();
if (!missingPartitionRanges.isEmpty()) {
for (UpdatePeriod period : missingPartitionRanges.keySet()) {
- for (TimePartition.TimePartitionRange range : missingPartitionRanges.get(period).getRanges()) {
+ for (TimePartitionRange range : missingPartitionRanges.get(period).getRanges()) {
nonExistingParts.add(range.toString());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/21102e6b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java
index 4c98d84..9d5d08b 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java
@@ -18,25 +18,36 @@
*/
package org.apache.lens.cube.metadata;
+import static org.testng.Assert.*;
+
+import java.util.Calendar;
import java.util.Date;
import org.apache.lens.api.LensException;
-import org.testng.Assert;
import org.testng.annotations.Test;
public class TestTimePartition {
+ public static final Date NOW = new Date();
+
@Test
public void test() throws LensException {
- Date now = new Date();
+ // Test for all update periods
for (UpdatePeriod up : UpdatePeriod.values()) {
- String nowStr = up.format().format(now);
- TimePartition nowPartition = TimePartition.of(up, now);
+ // Normal date object parsable
+ String nowStr = up.format().format(NOW);
+
+ // Create partition by date object or it's string representation -- both should be same.
+ TimePartition nowPartition = TimePartition.of(up, NOW);
TimePartition nowStrPartition = TimePartition.of(up, nowStr);
- Assert.assertEquals(nowPartition, nowStrPartition);
- Assert.assertTrue(nowPartition.next().after(nowPartition));
- Assert.assertTrue(nowPartition.previous().before(nowPartition));
- Assert.assertEquals(getLensExceptionFromPartitionParsing(up, "garbage").getMessage(),
+ assertEquals(nowPartition, nowStrPartition);
+
+ // Test next and previous
+ assertTrue(nowPartition.next().after(nowPartition));
+ assertTrue(nowPartition.previous().before(nowPartition));
+
+ // date parse failures should give lens exception
+ assertEquals(getLensExceptionFromPartitionParsing(up, "garbage").getMessage(),
TimePartition.getWrongUpdatePeriodMessage(up, "garbage"));
getLensExceptionFromPartitionParsing(up, (Date) null);
getLensExceptionFromPartitionParsing(up, (String) null);
@@ -48,7 +59,8 @@ public class TestTimePartition {
if (up.formatStr().equals(up2.formatStr())) {
continue;
}
- Assert.assertEquals(getLensExceptionFromPartitionParsing(up2, nowStr).getMessage(),
+ // Parsing a string representation with differnet update period should give lens exception.
+ assertEquals(getLensExceptionFromPartitionParsing(up2, nowStr).getMessage(),
TimePartition.getWrongUpdatePeriodMessage(up2, nowStr));
}
}
@@ -57,7 +69,7 @@ public class TestTimePartition {
private LensException getLensExceptionFromPartitionParsing(UpdatePeriod up, String dateStr) {
try {
TimePartition.of(up, dateStr);
- Assert.fail("Should have thrown LensException");
+ fail("Should have thrown LensException");
} catch (LensException e) {
return e;
}
@@ -67,10 +79,70 @@ public class TestTimePartition {
private LensException getLensExceptionFromPartitionParsing(UpdatePeriod up, Date date) {
try {
TimePartition.of(up, date);
- Assert.fail("Should have thrown LensException");
+ fail("Should have thrown LensException");
} catch (LensException e) {
return e;
}
return null; // redundant
}
+
+
+ public static Date timeAtDiff(Date date, UpdatePeriod period, int d) {
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(date);
+ if (period.equals(UpdatePeriod.QUARTERLY)) {
+ d *= 3;
+ }
+ cal.add(period.calendarField(), d);
+ return cal.getTime();
+ }
+
+ @Test
+ public void testTimeRange() throws LensException {
+ // test for all update periods
+ for (UpdatePeriod up : UpdatePeriod.values()) {
+ // create two partition of different time
+ TimePartition nowPartition = TimePartition.of(up, NOW);
+ TimePartition tenLater = TimePartition.of(up, timeAtDiff(NOW, up, 10));
+
+ // a.upto(b) == b.from(a)
+ TimePartitionRange range = nowPartition.rangeUpto(tenLater);
+ assertEquals(range, tenLater.rangeFrom(nowPartition));
+ // size check
+ assertEquals(range.size(), 10);
+ // test singleton range
+ assertEquals(nowPartition.singletonRange().size(), 1);
+ // test begin belongs to [begin, end) and end doesn't belong
+ assertTrue(range.contains(nowPartition));
+ assertFalse(range.contains(tenLater));
+ // test partition parsing for string arguments.
+ // a,b == [a,b)
+ // Other possible arguments: [a,b), [a,b], (a,b), (a,b]
+ String nowStr = nowPartition.getDateString();
+ String tenLaterStr = tenLater.getDateString();
+ assertEquals(TimePartitionRange.parseFrom(up, nowStr, tenLaterStr), range);
+ assertEquals(TimePartitionRange.parseFrom(up, "[" + nowStr, tenLaterStr + ")"), range);
+ assertEquals(TimePartitionRange.parseFrom(up, "[" + nowStr, tenLaterStr + "]"),
+ nowPartition.rangeUpto(tenLater.next()));
+ assertEquals(TimePartitionRange.parseFrom(up, "(" + nowStr, tenLaterStr + "]"),
+ nowPartition.next().rangeUpto(
+ tenLater.next()));
+ assertEquals(TimePartitionRange.parseFrom(up, "(" + nowStr, tenLaterStr + ")"),
+ nowPartition.next().rangeUpto(tenLater));
+ }
+ }
+
+ @Test(expectedExceptions = LensException.class)
+ public void testPartitionRangeValidity() throws LensException {
+ // begin and end partitions should follow begin <= end
+ TimePartition.of(UpdatePeriod.HOURLY, NOW)
+ .rangeFrom(TimePartition.of(UpdatePeriod.HOURLY, timeAtDiff(NOW, UpdatePeriod.HOURLY, 10)));
+ }
+
+ @Test(expectedExceptions = LensException.class)
+ public void testTimeRangeCreationWithDifferentUpdatePeriod() throws LensException {
+ // begin and end partitions should have same update period for range creation to succeed.
+ TimePartition.of(UpdatePeriod.HOURLY, NOW).rangeUpto(TimePartition.of(UpdatePeriod.DAILY, NOW));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/21102e6b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
index 50b75e3..c87050f 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
@@ -21,7 +21,7 @@ package org.apache.lens.cube.metadata.timeline;
import java.util.*;
import org.apache.lens.api.LensException;
-import org.apache.lens.cube.metadata.CubeMetastoreClient;
+import org.apache.lens.cube.metadata.TestTimePartition;
import org.apache.lens.cube.metadata.TimePartition;
import org.apache.lens.cube.metadata.UpdatePeriod;
@@ -31,11 +31,9 @@ import org.testng.annotations.Test;
import com.beust.jcommander.internal.Lists;
public class TestPartitionTimelines {
- CubeMetastoreClient client = null;
private static final String TABLE_NAME = "storage_fact";
- private static final UpdatePeriod PERIOD = UpdatePeriod.HOURLY;
+ public static final UpdatePeriod PERIOD = UpdatePeriod.HOURLY;
private static final String PART_COL = "pt";
- private static final Date DATE = new Date();
private static final List<Class<? extends PartitionTimeline>> TIMELINE_IMPLEMENTATIONS = Arrays.asList(
StoreAllPartitionTimeline.class,
EndsAndHolesPartitionTimeline.class,
@@ -60,7 +58,8 @@ public class TestPartitionTimelines {
final List<TimePartition> addedPartitions = Lists.newArrayList();
for (int i = 0; i < 200; i++) {
int randomInt = randomGenerator.nextInt(100) - 50;
- TimePartition part = TimePartition.of(PERIOD, timeAtHourDiff(randomInt));
+ TimePartition part = TimePartition.of(PERIOD, TestTimePartition.timeAtDiff(TestTimePartition.NOW, PERIOD,
+ randomInt));
addedPartitions.add(part);
for (PartitionTimeline timeline : timelines) {
timeline.add(part);
@@ -98,12 +97,6 @@ public class TestPartitionTimelines {
}
}
- private Date timeAtHourDiff(int d) {
- Calendar cal = Calendar.getInstance();
- cal.setTime(DATE);
- cal.add(PERIOD.calendarField(), d);
- return cal.getTime();
- }
private <T extends PartitionTimeline> T getInstance(Class<T> clz) {
try {
@@ -116,22 +109,41 @@ public class TestPartitionTimelines {
}
private <T extends PartitionTimeline> void testPropertiesContract(Class<T> clz) throws LensException {
+ // Make two instances, one to modify, other to validate against
T inst1 = getInstance(clz);
T inst2 = getInstance(clz);
+ // whenever we'll init from props, timeline should become empty.
Map<String, String> props = inst1.toProperties();
Assert.assertTrue(inst2.initFromProperties(props));
+ // init from props of an empty timeline: should succeed and make the timeline empty
Assert.assertEquals(inst1, inst2);
Assert.assertTrue(inst1.isEmpty());
Assert.assertTrue(inst2.isEmpty());
- Assert.assertTrue(inst1.add(TimePartition.of(PERIOD, DATE)));
+ // Add single partition and test for non-equivalence
+ Assert.assertTrue(inst1.add(TimePartition.of(PERIOD, TestTimePartition.NOW)));
Assert.assertFalse(inst1.equals(inst2));
- Assert.assertTrue(inst2.add(TimePartition.of(PERIOD, DATE)));
+ // add same parittion in other timeline, test for equality
+ Assert.assertTrue(inst2.add(TimePartition.of(PERIOD, TestTimePartition.NOW)));
Assert.assertTrue(inst1.isConsistent());
Assert.assertTrue(inst2.isConsistent());
Assert.assertEquals(inst1, inst2);
+ // init with blank properties. Should become empty
Assert.assertTrue(inst2.initFromProperties(props));
Assert.assertFalse(inst1.equals(inst2));
+ // init from properties of timeline with single partition.
Assert.assertTrue(inst2.initFromProperties(inst1.toProperties()));
Assert.assertEquals(inst1, inst2);
+ // clear timelines
+ inst1.initFromProperties(props);
+ inst2.initFromProperties(props);
+ // Make sparse partition range in one, init other from its properties. Test equality.
+ for (int i = 0; i < 5000; i++) {
+ Assert.assertTrue(inst1.add(TimePartition.of(PERIOD, TestTimePartition.timeAtDiff(TestTimePartition.NOW, PERIOD,
+ i * 2))));
+ }
+ Assert.assertTrue(inst1.isConsistent());
+ inst2.initFromProperties(inst1.toProperties());
+ Assert.assertTrue(inst2.isConsistent());
+ Assert.assertEquals(inst1, inst2);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/21102e6b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index b2ae9b5..62b2c95 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -1001,10 +1001,10 @@ public class CubeTestSetup {
Assert.assertEquals(table.getParameters().get(
MetastoreUtil.getPartitionTimelineStorageClassKey(UpdatePeriod.YEARLY, "ttd2")),
EndsAndHolesPartitionTimeline.class.getCanonicalName());
- Assert.assertEquals(table.getParameters().get(
+ Assert.assertEquals(MetastoreUtil.getNamedStringValue(table.getParameters(),
MetastoreUtil.getPartitionInfoKeyPrefix(UpdatePeriod.HOURLY, "ttd") + "partitions"),
StringUtils.join(partitions, ","));
- Assert.assertEquals(table.getParameters().get(
+ Assert.assertEquals(MetastoreUtil.getNamedStringValue(table.getParameters(),
MetastoreUtil.getPartitionInfoKeyPrefix(UpdatePeriod.HOURLY, "ttd2") + "partitions"),
StringUtils.join(partitions, ","));
// Add all hourly partitions for TWO_DAYS_RANGE_BEFORE_4_DAYS
@@ -2045,7 +2045,7 @@ public class CubeTestSetup {
for (String p : Arrays.asList("et", "it", "pt")) {
String first = params.get(prefix + up + "." + p + "." + "first");
String latest = params.get(prefix + up + "." + p + "." + "latest");
- String holes = params.get(prefix + up + "." + p + "." + "holes");
+ String holes = MetastoreUtil.getNamedStringValue(params, prefix + up + "." + p + "." + "holes");
String storageClass = params.get(prefix + up + "." + p + "." + "storage.class");
Assert.assertNotNull(first);
Assert.assertNotNull(latest);