You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/03/27 09:49:20 UTC
incubator-lens git commit: LENS-412 : Fix missing partitions message
propagation across storages (Rajat Khandelwal via amareshwari)
Repository: incubator-lens
Updated Branches:
refs/heads/master c6c593c66 -> 42ffb4e1a
LENS-412 : Fix missing partitions message propagation across storages (Rajat Khandelwal via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/42ffb4e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/42ffb4e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/42ffb4e1
Branch: refs/heads/master
Commit: 42ffb4e1a837320c0014fae7ce0665fb182d97d6
Parents: c6c593c
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Fri Mar 27 14:19:09 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Mar 27 14:19:09 2015 +0530
----------------------------------------------------------------------
.../lens/cube/metadata/CubeMetastoreClient.java | 24 +-
.../lens/cube/metadata/StorageConstants.java | 10 +-
.../lens/cube/metadata/TimePartition.java | 76 ++++++
.../timeline/EndsAndHolesPartitionTimeline.java | 22 +-
.../metadata/timeline/PartitionTimeline.java | 32 +--
.../timeline/PartitionTimelineFactory.java | 6 +-
.../timeline/RangesPartitionTimeline.java | 243 +++++++++++++++++++
.../timeline/StoreAllPartitionTimeline.java | 8 +-
.../cube/parse/CandidateTablePruneCause.java | 7 +-
.../lens/cube/parse/CubeQueryConfUtil.java | 2 -
.../lens/cube/parse/CubeQueryContext.java | 2 +-
.../org/apache/lens/cube/parse/PruneCauses.java | 40 +--
.../lens/cube/parse/StorageTableResolver.java | 70 +++---
.../org/apache/lens/cube/parse/StorageUtil.java | 6 +-
.../timeline/TestPartitionTimelines.java | 75 +++++-
.../lens/cube/parse/TestCubeRewriter.java | 32 +--
16 files changed, 508 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 42bf98f..1835d2f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -44,7 +44,6 @@ import org.apache.thrift.TException;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-
import lombok.extern.apachecommons.CommonsLog;
/**
@@ -330,14 +329,19 @@ public class CubeMetastoreClient {
}
/** update partition timeline cache for deletion of time partition */
- public void updateForDeletion(String cubeTableName, String storageName, UpdatePeriod updatePeriod,
+ public boolean updateForDeletion(String cubeTableName, String storageName, UpdatePeriod updatePeriod,
Map<String, Date> timePartSpec) throws HiveException, LensException {
+ boolean updated = false;
for (Map.Entry<String, Date> entry : timePartSpec.entrySet()) {
- get(cubeTableName, storageName, updatePeriod, entry.getKey()).drop(TimePartition.of(
- updatePeriod, entry.getValue()));
+ TimePartition part = TimePartition.of(updatePeriod, entry.getValue());
+ if (!partitionExistsByFilter(cubeTableName, storageName, StorageConstants.getPartFilter(entry.getKey(),
+ part.getDateString()))) {
+ get(cubeTableName, storageName, updatePeriod, entry.getKey()).drop(part);
+ updated = true;
+ }
}
+ return updated;
}
-
}
@@ -937,8 +941,9 @@ public class CubeMetastoreClient {
} else {
// dropping fact partition
getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, null);
- partitionTimelineCache.updateForDeletion(cubeTableName, storageName, updatePeriod, timePartSpec);
- this.alterTablePartitionCache(storageTableName);
+ if (partitionTimelineCache.updateForDeletion(cubeTableName, storageName, updatePeriod, timePartSpec)) {
+ this.alterTablePartitionCache(storageTableName);
+ }
}
}
@@ -978,6 +983,11 @@ public class CubeMetastoreClient {
return partitionExists(storageTableName, getPartitionSpec(updatePeriod, partitionTimestamps));
}
+ public boolean partitionExistsByFilter(String cubeTableName, String storageName, String filter) throws HiveException {
+ return partitionExistsByFilter(MetastoreUtil.getStorageTableName(cubeTableName, Storage.getPrefix(storageName)),
+ filter);
+ }
+
public boolean partitionExistsByFilter(String storageTableName, String filter) throws HiveException {
int parts;
Table tbl = null;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
index aee9f7b..610d168 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
@@ -19,10 +19,10 @@
package org.apache.lens.cube.metadata;
-import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import java.util.Set;
public final class StorageConstants {
private StorageConstants() {
@@ -52,10 +52,8 @@ public final class StorageConstants {
*
* @return List
*/
- public static List<String> getPartitionsForLatest() {
- List<String> parts = new ArrayList<String>();
- parts.add(LATEST_PARTITION_VALUE);
- return parts;
+ public static Set<String> getPartitionsForLatest() {
+ return Collections.singleton(LATEST_PARTITION_VALUE);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
index 6a5b31d..b948467 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
@@ -21,6 +21,7 @@ package org.apache.lens.cube.metadata;
import java.text.ParseException;
import java.util.Calendar;
import java.util.Date;
+import java.util.Iterator;
import org.apache.lens.api.LensException;
@@ -123,4 +124,79 @@ public class TimePartition implements Comparable<TimePartition> {
protected static String getWrongUpdatePeriodMessage(UpdatePeriod up, String dateString) {
return String.format(UPDATE_PERIOD_WRONG_ERROR_MESSAGE, up, dateString);
}
+
+ public TimePartitionRange rangeUpto(TimePartition to) {
+ return new TimePartitionRange(this, to);
+ }
+
+ public TimePartitionRange rangeFrom(TimePartition from) {
+ return new TimePartitionRange(from, this);
+ }
+
+ public TimePartitionRange singletonRange() {
+ return rangeUpto(next());
+ }
+
+ /**
+ * Range of time partition. [begin,end). i.e. inclusive begin and exclusive end.
+ */
+ @Data
+ public static class TimePartitionRange implements Iterable<TimePartition> {
+ private TimePartition begin;
+ private TimePartition end;
+
+ public TimePartitionRange(TimePartition from, TimePartition to) {
+ this.begin = from;
+ this.end = to;
+ }
+
+ @Override
+ public String toString() {
+ return "[" + begin.getDateString() + ", " + end.getDateString() + ")";
+ }
+
+ /**
+ * returns TimePartition objects starting from begin and upto(excluding) end. interval of iteration is the update
+ * period of the partitions. Assumes both partitions have same update period.
+ */
+ @Override
+ public Iterator<TimePartition> iterator() {
+
+ return new Iterator<TimePartition>() {
+ TimePartition current = begin;
+
+ @Override
+ public boolean hasNext() {
+ return current.before(end);
+ }
+
+ @Override
+ public TimePartition next() {
+ TimePartition ret = current;
+ current = current.next();
+ return ret;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("remove not supported");
+ }
+ };
+ }
+
+ /**
+ * @param partition
+ * @return begin <= partition < end
+ */
+ public boolean contains(TimePartition partition) {
+ return !partition.before(begin) && partition.before(end);
+ }
+
+ /**
+ * @return if range is empty range.
+ */
+ public boolean isEmpty() {
+ return begin.equals(end);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
index 3e323e1..79e8a62 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
@@ -22,7 +22,6 @@ package org.apache.lens.cube.metadata.timeline;
import java.util.*;
import org.apache.lens.api.LensException;
-import org.apache.lens.cube.metadata.CubeMetastoreClient;
import org.apache.lens.cube.metadata.TimePartition;
import org.apache.lens.cube.metadata.UpdatePeriod;
import org.apache.lens.cube.parse.TimeRange;
@@ -48,13 +47,13 @@ public class EndsAndHolesPartitionTimeline extends PartitionTimeline {
private TreeSet<TimePartition> holes = Sets.newTreeSet();
private TimePartition latest;
- public EndsAndHolesPartitionTimeline(CubeMetastoreClient client, String storageTableName, UpdatePeriod updatePeriod,
+ public EndsAndHolesPartitionTimeline(String storageTableName, UpdatePeriod updatePeriod,
String partCol) {
- super(client, storageTableName, updatePeriod, partCol);
+ super(storageTableName, updatePeriod, partCol);
}
@Override
- public boolean add(TimePartition partition) throws LensException {
+ public boolean add(@NonNull TimePartition partition) throws LensException {
if (isEmpty()) {
// First partition being added
first = partition;
@@ -75,20 +74,7 @@ public class EndsAndHolesPartitionTimeline extends PartitionTimeline {
}
@Override
- public boolean add(@NonNull Collection<TimePartition> partitions) throws LensException {
- boolean result = true;
- for (TimePartition partition : partitions) {
- result &= add(partition);
- }
- // Can also return the failed to add items.
- return result;
- }
-
- @Override
- public boolean drop(TimePartition toDrop) throws LensException {
- if (morePartitionsExist(toDrop.getDateString())) {
- return true;
- }
+ public boolean drop(@NonNull TimePartition toDrop) throws LensException {
if (first.equals(latest) && first.equals(toDrop)) {
this.first = null;
this.latest = null;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
index 7eda58a..237164f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
@@ -22,9 +22,10 @@ package org.apache.lens.cube.metadata.timeline;
import java.util.*;
import org.apache.lens.api.LensException;
-import org.apache.lens.cube.metadata.*;
+import org.apache.lens.cube.metadata.MetastoreUtil;
+import org.apache.lens.cube.metadata.TimePartition;
+import org.apache.lens.cube.metadata.UpdatePeriod;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import com.google.common.collect.Maps;
@@ -45,7 +46,6 @@ import lombok.extern.apachecommons.CommonsLog;
@ToString(exclude = {"client"})
@CommonsLog
public abstract class PartitionTimeline implements Iterable<TimePartition> {
- private final CubeMetastoreClient client;
private final String storageTableName;
private final UpdatePeriod updatePeriod;
private final String partCol;
@@ -121,23 +121,6 @@ public abstract class PartitionTimeline implements Iterable<TimePartition> {
return result;
}
- /**
- * goes to metastore and queries if more partitions exist associated with (partCol = value) in storage table
- * #getStorageTableName for update period #getUpdatePeriod. This might be useful for implementations while
- * implementing drop.
- *
- * @param value
- * @return
- * @throws LensException
- */
- public boolean morePartitionsExist(String value) throws LensException {
- try {
- return getClient().partitionExistsByFilter(getStorageTableName(), StorageConstants.getPartFilter(getPartCol(),
- value));
- } catch (HiveException e) {
- throw new LensException(e);
- }
- }
/**
* Add partition to timeline
@@ -155,7 +138,14 @@ public abstract class PartitionTimeline implements Iterable<TimePartition> {
* @return whether add was successful
* @throws LensException
*/
- public abstract boolean add(@NonNull Collection<TimePartition> partitions) throws LensException;
+ public boolean add(@NonNull Collection<TimePartition> partitions) throws LensException {
+ boolean result = true;
+ for (TimePartition partition : partitions) {
+ result &= add(partition);
+ }
+ // Can also return the failed to add items.
+ return result;
+ }
/**
* drop partition.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java
index 5626a03..b018a1a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java
@@ -49,11 +49,11 @@ public final class PartitionTimelineFactory {
updatePeriod, partitionColumn));
Class<? extends PartitionTimeline> clz = (Class<? extends PartitionTimeline>) Class.forName(storageClassName);
Constructor<? extends PartitionTimeline> constructor = clz.getConstructor(
- CubeMetastoreClient.class, String.class, UpdatePeriod.class, String.class);
+ String.class, UpdatePeriod.class, String.class);
return constructor.newInstance(
- client, storageTable, updatePeriod, partitionColumn);
+ storageTable, updatePeriod, partitionColumn);
} catch (Exception e) {
- return new EndsAndHolesPartitionTimeline(client, storageTable, updatePeriod, partitionColumn);
+ return new EndsAndHolesPartitionTimeline(storageTable, updatePeriod, partitionColumn);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java
new file mode 100644
index 0000000..fb2d0a8
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata.timeline;
+
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.cube.metadata.TimePartition;
+import org.apache.lens.cube.metadata.UpdatePeriod;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import lombok.Data;
+import lombok.ToString;
+
+/**
+ * One implementation of PartitionTimeline that stores ranges of partition presence, Basically a list of tuples each
+ * tuple represents a range of presence. range is of the form [from, end) i.e. including the first element and excluding
+ * the second element of the tuple
+ */
+@Data
+@ToString(callSuper = true)
+public class RangesPartitionTimeline extends PartitionTimeline {
+ private List<TimePartition.TimePartitionRange> ranges = Lists.newArrayList();
+
+ public RangesPartitionTimeline(String storageTableName, UpdatePeriod updatePeriod,
+ String partCol) {
+ super(storageTableName, updatePeriod, partCol);
+ }
+
+ @Override
+ public boolean add(TimePartition partition) throws LensException {
+ int ind = getStrictlyAfterIndex(partition);
+ int added = 0;
+ if (ind > 0) {
+ if (ranges.get(ind - 1).contains(partition)) {
+ return true;
+ }
+ if (ranges.get(ind - 1).getEnd().equals(partition)) {
+ added++;
+ ranges.get(ind - 1).setEnd(partition.next());
+ }
+ }
+ if (ind < ranges.size()) {
+ if (partition.equals(ranges.get(ind).getBegin().previous())) {
+ added++;
+ ranges.get(ind).setBegin(partition);
+ }
+ }
+ switch (added) {
+ case 0:
+ ranges.add(ind, partition.singletonRange());
+ break;
+ case 2:
+ ranges.get(ind - 1).setEnd(ranges.get(ind).getEnd());
+ ranges.remove(ind);
+ break;
+ case 1:
+ // Nothing needs to be done.
+ default:
+ break;
+
+ }
+ return true;
+ }
+
+ private int getStrictlyAfterIndex(TimePartition part) {
+ int start = 0;
+ int end = getRanges().size();
+ int mid;
+ while (end - start > 0) {
+ mid = (start + end) / 2;
+ if (ranges.get(mid).getBegin().after(part)) {
+ end = mid;
+ } else {
+ start = mid + 1;
+ }
+ }
+ return end;
+ }
+
+ private void mergeRanges() {
+ for (int i = 0; i < ranges.size() - 1; i++) {
+ if (ranges.get(i).getEnd().equals(ranges.get(i + 1).getBegin())) {
+ TimePartition.TimePartitionRange removed = ranges.remove(i + 1);
+ ranges.get(i).setEnd(removed.getEnd());
+ i--; // check again at same index
+ }
+ }
+ }
+
+ @Override
+ public boolean drop(TimePartition toDrop) throws LensException {
+ int ind = getStrictlyAfterIndex(toDrop);
+ if (ind == 0) {
+ return true; // nothing to do
+ }
+ if (ranges.get(ind - 1).getBegin().equals(toDrop)) {
+ ranges.get(ind - 1).setBegin(toDrop.next());
+ } else if (ranges.get(ind - 1).getEnd().previous().equals(toDrop)) {
+ ranges.get(ind - 1).setEnd(toDrop);
+ } else {
+ TimePartition end = ranges.get(ind - 1).getEnd();
+ ranges.get(ind - 1).setEnd(toDrop);
+ ranges.add(ind, toDrop.next().rangeUpto(end));
+ }
+ if (ranges.get(ind - 1).isEmpty()) {
+ ranges.remove(ind - 1);
+ }
+ return true;
+ }
+
+
+ @Override
+ public TimePartition latest() {
+ if (isEmpty()) {
+ return null;
+ }
+ return ranges.get(ranges.size() - 1).getEnd().previous();
+ }
+
+ @Override
+ public Map<String, String> toProperties() {
+ HashMap<String, String> ret = Maps.newHashMap();
+ if (isEmpty()) {
+ return ret;
+ }
+ StringBuilder sb = new StringBuilder();
+ String sep = "";
+ for (TimePartition.TimePartitionRange range : ranges) {
+ sb.append(sep);
+ sep = ",";
+ sb.append(range.getBegin()).append(sep).append(range.getEnd());
+ }
+ ret.put("ranges", sb.toString());
+ return ret;
+ }
+
+ @Override
+ public boolean initFromProperties(Map<String, String> properties) throws LensException {
+ ranges.clear();
+ String rangesStr = properties.get("ranges");
+ if (!Strings.isNullOrEmpty(rangesStr)) {
+ String[] split = rangesStr.split("\\s*,\\s*");
+ if (split.length % 2 == 1) {
+ throw new LensException("Ranges incomplete");
+ }
+ for (int i = 0; i < split.length; i += 2) {
+ ranges.add(TimePartition.of(getUpdatePeriod(), split[i]).rangeUpto(TimePartition.of(getUpdatePeriod(),
+ split[i + 1])));
+ }
+ }
+ return isConsistent();
+ }
+
+
+ public boolean isEmpty() {
+ return ranges.isEmpty();
+ }
+
+ @Override
+ public boolean isConsistent() {
+ if (isEmpty()) {
+ return true;
+ }
+ if (!ranges.get(0).getBegin().before(ranges.get(0).getEnd())) {
+ return false;
+ }
+ for (int i = 0; i < ranges.size() - 1; i++) {
+ if (!ranges.get(i).getEnd().before(ranges.get(i + 1).getBegin())) {
+ return false;
+ }
+ if (!ranges.get(i + 1).getBegin().before(ranges.get(i + 1).getEnd())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public boolean exists(TimePartition toCheck) {
+ if (isEmpty()) {
+ return false;
+ }
+ for (TimePartition.TimePartitionRange range : ranges) {
+ if (range.contains(toCheck)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public Iterator<TimePartition> iterator() {
+
+ return new Iterator<TimePartition>() {
+ Iterator<TimePartition.TimePartitionRange> uber = ranges.iterator();
+ Iterator<TimePartition> cur = null;
+
+ @Override
+ public boolean hasNext() {
+ if (cur == null || !cur.hasNext()) {
+ if (!uber.hasNext()) {
+ return false;
+ }
+ cur = uber.next().iterator();
+ }
+ return cur.hasNext();
+ }
+
+ @Override
+ public TimePartition next() {
+ return cur.next();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
index 8f8b03a..d6ee0a1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
@@ -21,7 +21,6 @@ package org.apache.lens.cube.metadata.timeline;
import java.util.*;
import org.apache.lens.api.LensException;
-import org.apache.lens.cube.metadata.CubeMetastoreClient;
import org.apache.lens.cube.metadata.TimePartition;
import org.apache.lens.cube.metadata.UpdatePeriod;
@@ -42,9 +41,9 @@ import lombok.ToString;
public class StoreAllPartitionTimeline extends PartitionTimeline {
TreeSet<TimePartition> allPartitions;
- public StoreAllPartitionTimeline(CubeMetastoreClient client, String storageTableName,
+ public StoreAllPartitionTimeline(String storageTableName,
UpdatePeriod updatePeriod, String partCol) {
- super(client, storageTableName, updatePeriod, partCol);
+ super(storageTableName, updatePeriod, partCol);
allPartitions = Sets.newTreeSet();
}
@@ -60,9 +59,6 @@ public class StoreAllPartitionTimeline extends PartitionTimeline {
@Override
public boolean drop(@NonNull TimePartition toDrop) throws LensException {
- if (morePartitionsExist(toDrop.getDateString())) {
- return true;
- }
return allPartitions.remove(toDrop);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
index bfeff4f..bc9ef93 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
@@ -22,6 +22,7 @@ import java.util.*;
import org.codehaus.jackson.annotate.JsonWriteNullProperties;
+import com.google.common.collect.Sets;
import lombok.Data;
import lombok.NoArgsConstructor;
@@ -95,7 +96,7 @@ public class CandidateTablePruneCause {
// missing partitions for cube table
MISSING_PARTITIONS("Missing partitions for the cube table: %s") {
Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) {
- List<List<String>> missingPartitions = new ArrayList<List<String>>();
+ Set<Set<String>> missingPartitions = Sets.newHashSet();
for (CandidateTablePruneCause cause : causes) {
missingPartitions.add(cause.getMissingPartitions());
}
@@ -183,7 +184,7 @@ public class CandidateTablePruneCause {
private Map<String, SkipStorageCause> storageCauses;
// populated only incase of missing partitions cause
- private List<String> missingPartitions;
+ private Set<String> missingPartitions;
// populated only incase of missing update periods cause
private List<String> missingUpdatePeriods;
// populated in case of missing columns
@@ -215,7 +216,7 @@ public class CandidateTablePruneCause {
return columnNotFound(colList);
}
- public static CandidateTablePruneCause missingPartitions(List<String> nonExistingParts) {
+ public static CandidateTablePruneCause missingPartitions(Set<String> nonExistingParts) {
CandidateTablePruneCause cause =
new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_PARTITIONS);
cause.setMissingPartitions(nonExistingParts);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
index c3142cd..a6374f6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
@@ -44,7 +44,6 @@ public final class CubeQueryConfUtil {
public static final String DRIVER_SUPPORTED_STORAGES = "lens.cube.query.driver." + "supported.storages";
public static final String FAIL_QUERY_ON_PARTIAL_DATA = "lens.cube.query.fail.if.data.partial";
public static final String NON_EXISTING_PARTITIONS = "lens.cube.query.nonexisting.partitions";
- public static final String ADD_NON_EXISTING_PARTITIONS = "lens.cube.query.add.nonexisting.partitions";
public static final String ENABLE_MULTI_TABLE_SELECT = "lens.cube.query.enable.multi.table.select";
public static final String QUERY_MAX_INTERVAL = "lens.cube.query.max.interval";
public static final String PROCESS_TIME_PART_COL = "lens.cube.query.process.time" + ".partition.column";
@@ -59,7 +58,6 @@ public final class CubeQueryConfUtil {
public static final int DEFAULT_LOOK_AHEAD_PT_PARTS = 1;
public static final boolean DEFAULT_ENABLE_GROUP_BY_TO_SELECT = false;
public static final boolean DEFAULT_ENABLE_SELECT_TO_GROUPBY = false;
- public static final boolean DEFAULT_ADD_NON_EXISTING_PARTITIONS = false;
public static final boolean DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL = true;
public static String getLookAheadPTPartsKey(UpdatePeriod interval) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index e06022c..7ea67f4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -641,7 +641,7 @@ public class CubeQueryContext {
}
}
- void setNonexistingParts(Map<String, List<String>> nonExistingParts) throws SemanticException {
+ void setNonexistingParts(Map<String, Set<String>> nonExistingParts) throws SemanticException {
if (!nonExistingParts.isEmpty()) {
ByteArrayOutputStream out = null;
String partsStr;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java
index ae9e013..32ef421 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java
@@ -18,10 +18,7 @@
*/
package org.apache.lens.cube.parse;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import org.apache.lens.cube.metadata.AbstractCubeTable;
import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
@@ -30,6 +27,7 @@ import org.apache.commons.lang.StringUtils;
import org.codehaus.jackson.annotate.JsonWriteNullProperties;
+import com.google.common.collect.Maps;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
@@ -39,6 +37,21 @@ public class PruneCauses<T extends AbstractCubeTable> extends HashMap<T, List<Ca
@Getter(lazy = true)
private final HashMap<CandidateTablePruneCause, List<T>> reversed = reverse();
@Getter(lazy = true)
+ private final HashMap<String, List<CandidateTablePruneCause>> compact = computeCompact();
+
+ private HashMap<String, List<CandidateTablePruneCause>> computeCompact() {
+ HashMap<String, List<CandidateTablePruneCause>> detailedMessage = Maps.newHashMap();
+ for (Map.Entry<CandidateTablePruneCause, List<T>> entry : getReversed().entrySet()) {
+ String key = StringUtils.join(entry.getValue(), ",");
+ if (detailedMessage.get(key) == null) {
+ detailedMessage.put(key, new ArrayList<CandidateTablePruneCause>());
+ }
+ detailedMessage.get(key).add(entry.getKey());
+ }
+ return detailedMessage;
+ }
+
+ @Getter(lazy = true)
private final BriefAndDetailedError jsonObject = toJsonObject();
public void addPruningMsg(T table, CandidateTablePruneCause msg) {
@@ -62,16 +75,7 @@ public class PruneCauses<T extends AbstractCubeTable> extends HashMap<T, List<Ca
}
public BriefAndDetailedError toJsonObject() {
- final HashMap<String, List<CandidateTablePruneCause>> detailedMessage
- = new HashMap<String, List<CandidateTablePruneCause>>();
- for (Map.Entry<CandidateTablePruneCause, List<T>> entry : getReversed().entrySet()) {
- String key = StringUtils.join(entry.getValue(), ",");
- if (detailedMessage.get(key) == null) {
- detailedMessage.put(key, new ArrayList<CandidateTablePruneCause>());
- }
- detailedMessage.get(key).add(entry.getKey());
- }
- return new BriefAndDetailedError(getBriefCause(), detailedMessage);
+ return new BriefAndDetailedError(getBriefCause(), getCompact());
}
public String getBriefCause() {
@@ -81,10 +85,10 @@ public class PruneCauses<T extends AbstractCubeTable> extends HashMap<T, List<Ca
maxCause = cause.getCause();
}
}
- Map<CandidateTablePruneCause, List<T>> maxCauseMap = new HashMap<CandidateTablePruneCause, List<T>>();
- for (Map.Entry<CandidateTablePruneCause, List<T>> entry : getReversed().entrySet()) {
- if (entry.getKey().getCause().compareTo(maxCause) == 0) {
- maxCauseMap.put(entry.getKey(), entry.getValue());
+ Map<CandidateTablePruneCause, String> maxCauseMap = Maps.newHashMap();
+ for (Map.Entry<CandidateTablePruneCause, List<T>> entry: getReversed().entrySet()) {
+ if (entry.getKey().getCause().equals(maxCause)) {
+ maxCauseMap.put(entry.getKey(), StringUtils.join(entry.getValue(), ","));
}
}
return maxCause.getBriefError(maxCauseMap.keySet());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 23fd5a6..6e63483 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -25,6 +25,7 @@ import java.util.*;
import org.apache.lens.api.LensException;
import org.apache.lens.cube.metadata.*;
+import org.apache.lens.cube.metadata.timeline.RangesPartitionTimeline;
import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCause;
import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode;
@@ -38,6 +39,9 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.util.ReflectionUtils;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
/**
* Resolve storages and partitions of all candidate tables and prunes candidate tables with missing storages or
* partitions.
@@ -55,8 +59,7 @@ class StorageTableResolver implements ContextRewriter {
new HashMap<CubeFactTable, Map<UpdatePeriod, Set<String>>>();
private String processTimePartCol = null;
private final UpdatePeriod maxInterval;
- private final boolean populateNonExistingParts;
- private final Map<String, List<String>> nonExistingPartitions = new HashMap<String, List<String>>();
+ private final Map<String, Set<String>> nonExistingPartitions = new HashMap<String, Set<String>>();
private TimeRangeWriter rangeWriter;
private DateFormat partWhereClauseFormat = null;
private PHASE phase;
@@ -82,13 +85,6 @@ class StorageTableResolver implements ContextRewriter {
this.supportedStorages = getSupportedStorages(conf);
this.allStoragesSupported = (supportedStorages == null);
this.failOnPartialData = conf.getBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false);
- if (!failOnPartialData) {
- this.populateNonExistingParts = true;
- } else {
- this.populateNonExistingParts =
- conf.getBoolean(CubeQueryConfUtil.ADD_NON_EXISTING_PARTITIONS,
- CubeQueryConfUtil.DEFAULT_ADD_NON_EXISTING_PARTITIONS);
- }
String str = conf.get(CubeQueryConfUtil.VALID_STORAGE_DIM_TABLES);
validDimTables = StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
this.processTimePartCol = conf.get(CubeQueryConfUtil.PROCESS_TIME_PART_COL);
@@ -320,10 +316,10 @@ class StorageTableResolver implements ContextRewriter {
CandidateFact cfact = i.next();
List<FactPartition> answeringParts = new ArrayList<FactPartition>();
HashMap<String, SkipStorageCause> skipStorageCauses = new HashMap<String, SkipStorageCause>();
- List<String> nonExistingParts = new ArrayList<String>();
+ Map<UpdatePeriod, RangesPartitionTimeline> missingPartitionRanges = Maps.newHashMap();
boolean noPartsForRange = false;
for (TimeRange range : cubeql.getTimeRanges()) {
- Set<FactPartition> rangeParts = getPartitions(cfact.fact, range, skipStorageCauses, nonExistingParts);
+ Set<FactPartition> rangeParts = getPartitions(cfact.fact, range, skipStorageCauses, missingPartitionRanges);
if (rangeParts == null || rangeParts.isEmpty()) {
LOG.info("No partitions for range:" + range);
noPartsForRange = true;
@@ -335,6 +331,14 @@ class StorageTableResolver implements ContextRewriter {
cfact.getRangeToWhereClause().put(range, rangeWriter.getTimeRangeWhereClause(cubeql,
cubeql.getAliasForTabName(cubeql.getCube().getName()), rangeParts));
}
+ Set<String> nonExistingParts = Sets.newHashSet();
+ if (!missingPartitionRanges.isEmpty()) {
+ for (UpdatePeriod period : missingPartitionRanges.keySet()) {
+ for (TimePartition.TimePartitionRange range : missingPartitionRanges.get(period).getRanges()) {
+ nonExistingParts.add(range.toString());
+ }
+ }
+ }
if (!nonExistingParts.isEmpty()) {
addNonExistingParts(cfact.fact.getName(), nonExistingParts);
}
@@ -343,24 +347,19 @@ class StorageTableResolver implements ContextRewriter {
+ cubeql.getTimeRanges());
/*
* This fact is getting discarded because of any of following reasons:
- * 1. Storage tables are not partitioned by timedim partition column
- * 2. Has missing partitions, and CubeQueryConfUtil.ADD_NON_EXISTING_PARTITIONS is true - which can populate
- * all missing partitions
- * 3. Has missing partitions, and CubeQueryConfUtil.ADD_NON_EXISTING_PARTITIONS is false - will populate only
- * the first missing partition.
- * 4. Storage tables do not have the update period for the timerange queried.
+ * 1. Has missing partitions
+ * 2. All Storage tables were skipped for some reasons.
+ * 3. Storage tables do not have the update period for the timerange queried.
*/
- if (!skipStorageCauses.isEmpty()) {
+ if (!nonExistingParts.isEmpty()) {
+ cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.missingPartitions(nonExistingParts));
+ } else if (!skipStorageCauses.isEmpty()) {
CandidateTablePruneCause cause = CandidateTablePruneCause.noCandidateStorages(skipStorageCauses);
cubeql.addFactPruningMsgs(cfact.fact, cause);
} else {
- if (!nonExistingParts.isEmpty()) {
- cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.missingPartitions(nonExistingParts));
- } else {
- CandidateTablePruneCause cause =
- new CandidateTablePruneCause(CandidateTablePruneCode.NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE);
- cubeql.addFactPruningMsgs(cfact.fact, cause);
- }
+ CandidateTablePruneCause cause =
+ new CandidateTablePruneCause(CandidateTablePruneCode.NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE);
+ cubeql.addFactPruningMsgs(cfact.fact, cause);
}
i.remove();
continue;
@@ -385,14 +384,16 @@ class StorageTableResolver implements ContextRewriter {
}
}
- void addNonExistingParts(String name, List<String> nonExistingParts) {
+
+ void addNonExistingParts(String name, Set<String> nonExistingParts) {
nonExistingPartitions.put(name, nonExistingParts);
}
private Set<FactPartition> getPartitions(CubeFactTable fact, TimeRange range,
- HashMap<String, SkipStorageCause> skipStorageCauses, List<String> nonExistingParts) throws SemanticException {
+ HashMap<String, SkipStorageCause> skipStorageCauses,
+ Map<UpdatePeriod, RangesPartitionTimeline> nonExistingParts) throws SemanticException {
try {
- return getPartitions(fact, range, getValidUpdatePeriods(fact), this.populateNonExistingParts, skipStorageCauses,
+ return getPartitions(fact, range, getValidUpdatePeriods(fact), true, skipStorageCauses,
nonExistingParts);
} catch (Exception e) {
throw new SemanticException(e);
@@ -400,7 +401,8 @@ class StorageTableResolver implements ContextRewriter {
}
private Set<FactPartition> getPartitions(CubeFactTable fact, TimeRange range, TreeSet<UpdatePeriod> updatePeriods,
- boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses, List<String> nonExistingParts)
+ boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses,
+ Map<UpdatePeriod, RangesPartitionTimeline> nonExistingParts)
throws Exception {
Set<FactPartition> partitions = new TreeSet<FactPartition>();
if (getPartitions(fact, range.getFromDate(), range.getToDate(), range.getPartitionColumn(), partitions,
@@ -413,7 +415,8 @@ class StorageTableResolver implements ContextRewriter {
private boolean getPartitions(CubeFactTable fact, Date fromDate, Date toDate, String partCol,
Set<FactPartition> partitions, TreeSet<UpdatePeriod> updatePeriods,
- boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses, List<String> nonExistingParts)
+ boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses,
+ Map<UpdatePeriod, RangesPartitionTimeline> nonExistingParts)
throws Exception {
LOG.info("getPartitions for " + fact + " from fromDate:" + fromDate + " toDate:" + toDate);
if (fromDate.equals(toDate) || fromDate.after(toDate)) {
@@ -526,10 +529,13 @@ class StorageTableResolver implements ContextRewriter {
if (!getPartitions(fact, dt, nextDt, partCol, partitions, newset, false, skipStorageCauses,
nonExistingParts)) {
- // Add non existing partitions for all cases of whether we populate all non existing or not.
LOG.info("Adding non existing partition" + part);
- nonExistingParts.add(part.getPartString());
if (addNonExistingParts) {
+ // Add non existing partitions for all cases of whether we populate all non existing or not.
+ if (!nonExistingParts.containsKey(part.getPeriod())) {
+ nonExistingParts.put(part.getPeriod(), new RangesPartitionTimeline(null, null, null));
+ }
+ nonExistingParts.get(part.getPeriod()).add(TimePartition.of(part.getPeriod(), dt));
if (!failOnPartialData) {
partitions.add(part);
// add all storage tables as the answering tables
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java
index 24d9340..0704171 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java
@@ -33,20 +33,20 @@ public final class StorageUtil {
private static final Log LOG = LogFactory.getLog(StorageUtil.class.getName());
- public static String getWherePartClause(String timeDimName, String tableName, List<String> parts) {
+ public static String getWherePartClause(String timeDimName, String tableName, Collection<String> parts) {
if (parts.size() == 0) {
return "";
}
StringBuilder partStr = new StringBuilder();
String sep = "";
- for (int i = 0; i < parts.size(); i++) {
+ for (String part : parts) {
partStr.append(sep);
partStr.append("(");
partStr.append(tableName != null ? tableName : "%s");
partStr.append(".");
partStr.append(timeDimName);
partStr.append(" = '");
- partStr.append(parts.get(i));
+ partStr.append(part);
partStr.append("'");
partStr.append(")");
sep = " OR ";
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
index 0027e64..50b75e3 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
@@ -18,8 +18,7 @@
*/
package org.apache.lens.cube.metadata.timeline;
-import java.util.Date;
-import java.util.Map;
+import java.util.*;
import org.apache.lens.api.LensException;
import org.apache.lens.cube.metadata.CubeMetastoreClient;
@@ -29,23 +28,87 @@ import org.apache.lens.cube.metadata.UpdatePeriod;
import org.testng.Assert;
import org.testng.annotations.Test;
+import com.beust.jcommander.internal.Lists;
+
public class TestPartitionTimelines {
CubeMetastoreClient client = null;
private static final String TABLE_NAME = "storage_fact";
private static final UpdatePeriod PERIOD = UpdatePeriod.HOURLY;
private static final String PART_COL = "pt";
private static final Date DATE = new Date();
+ private static final List<Class<? extends PartitionTimeline>> TIMELINE_IMPLEMENTATIONS = Arrays.asList(
+ StoreAllPartitionTimeline.class,
+ EndsAndHolesPartitionTimeline.class,
+ RangesPartitionTimeline.class
+ );
@Test
public void testPropertiesContractsForAllSubclasses() throws LensException {
- testPropertiesContract(StoreAllPartitionTimeline.class);
- testPropertiesContract(EndsAndHolesPartitionTimeline.class);
+ for (Class<? extends PartitionTimeline> clazz : TIMELINE_IMPLEMENTATIONS) {
+ testPropertiesContract(clazz);
+ }
+ }
+
+ @Test
+ public void testEquivalence() throws LensException {
+ for (int j = 0; j < 10; j++) {
+ Random randomGenerator = new Random();
+ List<PartitionTimeline> timelines = Lists.newArrayList();
+ for (Class<? extends PartitionTimeline> clazz : TIMELINE_IMPLEMENTATIONS) {
+ timelines.add(getInstance(clazz));
+ }
+ final List<TimePartition> addedPartitions = Lists.newArrayList();
+ for (int i = 0; i < 200; i++) {
+ int randomInt = randomGenerator.nextInt(100) - 50;
+ TimePartition part = TimePartition.of(PERIOD, timeAtHourDiff(randomInt));
+ addedPartitions.add(part);
+ for (PartitionTimeline timeline : timelines) {
+ timeline.add(part);
+ }
+ }
+ Iterator<TimePartition> sourceOfTruth = timelines.get(0).iterator();
+ List<Iterator<TimePartition>> otherIterators = Lists.newArrayList();
+ for (int i = 1; i < TIMELINE_IMPLEMENTATIONS.size() - 1; i++) {
+ otherIterators.add(timelines.get(i).iterator());
+ }
+ while (sourceOfTruth.hasNext()) {
+ TimePartition cur = sourceOfTruth.next();
+ for (Iterator<TimePartition> iterator : otherIterators) {
+ Assert.assertTrue(iterator.hasNext());
+ Assert.assertEquals(iterator.next(), cur);
+ }
+ }
+ for (Iterator<TimePartition> iterator : otherIterators) {
+ Assert.assertFalse(iterator.hasNext());
+ }
+ Collections.shuffle(addedPartitions);
+ Iterator<TimePartition> iter = addedPartitions.iterator();
+ while (iter.hasNext()) {
+ TimePartition part = iter.next();
+ iter.remove();
+ if (!addedPartitions.contains(part)) {
+ for (PartitionTimeline timeline : timelines) {
+ timeline.drop(part);
+ }
+ }
+ }
+ for (PartitionTimeline timeline : timelines) {
+ Assert.assertTrue(timeline.isEmpty());
+ }
+ }
+ }
+
+ private Date timeAtHourDiff(int d) {
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(DATE);
+ cal.add(PERIOD.calendarField(), d);
+ return cal.getTime();
}
private <T extends PartitionTimeline> T getInstance(Class<T> clz) {
try {
- return clz.getConstructor(CubeMetastoreClient.class, String.class, UpdatePeriod.class, String.class)
- .newInstance(client, TABLE_NAME, PERIOD, PART_COL);
+ return clz.getConstructor(String.class, UpdatePeriod.class, String.class)
+ .newInstance(TABLE_NAME, PERIOD, PART_COL);
} catch (Exception e) {
e.printStackTrace();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 4278229..f9a3762 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -109,7 +109,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
compareQueries(expected, hqlQuery);
conf.setBoolean(CubeQueryConfUtil.LIGHTEST_FACT_FIRST, true);
- conf.setBoolean(CubeQueryConfUtil.ADD_NON_EXISTING_PARTITIONS, true);
+
SemanticException th = getSemanticExceptionInRewrite(
"select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf);
Assert.assertEquals(th.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.NO_CANDIDATE_FACT_AVAILABLE.getErrorCode());
@@ -122,24 +122,6 @@ public class TestCubeRewriter extends TestQueryRewrite {
Assert.assertEquals(pruneCauses.getDetails().get("testfact").size(), 1);
Assert.assertEquals(pruneCauses.getDetails().get("testfact").iterator().next().getCause(),
CandidateTablePruneCode.MISSING_PARTITIONS);
-
- // Error should be no missing partitions with first missing partition populated for each update period
- conf.setBoolean(CubeQueryConfUtil.ADD_NON_EXISTING_PARTITIONS, false);
- th = getSemanticExceptionInRewrite(
- "select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf);
- Assert.assertEquals(th.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.NO_CANDIDATE_FACT_AVAILABLE.getErrorCode());
- pruneCauses = extractPruneCause(th);
- Assert.assertEquals(
- pruneCauses.getBrief().substring(0, CandidateTablePruneCode.MISSING_PARTITIONS.errorFormat.length() - 3),
- CandidateTablePruneCode.MISSING_PARTITIONS.errorFormat.substring(0,
- CandidateTablePruneCode.MISSING_PARTITIONS.errorFormat.length() - 3)
- );
- Assert.assertEquals(pruneCauses.getDetails().get("testfact").size(), 1);
- Assert.assertEquals(pruneCauses.getDetails().get("testfact").iterator().next().getCause(),
- CandidateTablePruneCode.MISSING_PARTITIONS);
- Assert.assertEquals(pruneCauses.getDetails().get("testfactmonthly").size(), 1);
- Assert.assertEquals(pruneCauses.getDetails().get("testfactmonthly").iterator().next().getCause(),
- CandidateTablePruneCode.NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE);
}
@Test
@@ -858,12 +840,20 @@ public class TestCubeRewriter extends TestQueryRewrite {
CandidateTablePruneCode.MISSING_PARTITIONS);
Assert.assertEquals(pruneCauses.getDetails().get("testfactmonthly").iterator().next().getCause(),
CandidateTablePruneCode.MISSING_PARTITIONS);
- Assert.assertEquals(pruneCauses.getDetails().get("testfact2_raw,testfact2").iterator().next().getCause(),
+ Assert.assertEquals(pruneCauses.getDetails().get("testfact2").iterator().next().getCause(),
CandidateTablePruneCode.MISSING_PARTITIONS);
+ Assert.assertEquals(pruneCauses.getDetails().get("testfact2_raw").iterator().next().getCause(),
+ CandidateTablePruneCode.MISSING_PARTITIONS);
Assert.assertEquals(pruneCauses.getDetails().get("cheapfact").iterator().next().getCause(),
CandidateTablePruneCode.NO_CANDIDATE_STORAGES);
- Assert.assertEquals(pruneCauses.getDetails().get("summary1,summary2,summary3,summary4").iterator().next()
+ Assert.assertEquals(pruneCauses.getDetails().get("summary1,summary2,summary3").iterator().next().getCause(),
+ CandidateTablePruneCode.MISSING_PARTITIONS);
+ Assert.assertEquals(pruneCauses.getDetails().get("summary4").iterator().next()
.getCause(), CandidateTablePruneCode.NO_CANDIDATE_STORAGES);
+ Assert.assertEquals(pruneCauses.getDetails().get("summary4").iterator().next()
+ .getStorageCauses().values().iterator().next().getCause(), SkipStorageCode.PART_COL_DOES_NOT_EXIST);
+ Assert.assertEquals(pruneCauses.getDetails().get("summary4").iterator().next()
+ .getStorageCauses().values().iterator().next().getNonExistantPartCols(), Arrays.asList("dt"));
}
@Test