You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/03/10 07:55:53 UTC
[3/3] incubator-lens git commit: LENS-281 : Create Partition Timeline
and keep it cached and stored as storage table properties (Rajat Khandelwal
via amareshwari)
LENS-281 : Create Partition Timeline and keep it cached and stored as storage table properties (Rajat Khandelwal via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/803448a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/803448a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/803448a1
Branch: refs/heads/master
Commit: 803448a18e901a5c3dd9e4a70aa03935be05790b
Parents: 683516c
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Tue Mar 10 12:25:19 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Mar 10 12:25:19 2015 +0530
----------------------------------------------------------------------
.../metadata/CaseInsensitiveStringHashMap.java | 55 +++
.../lens/cube/metadata/CubeFactTable.java | 6 +-
.../lens/cube/metadata/CubeMetastoreClient.java | 472 +++++++++++++++---
.../lens/cube/metadata/FactPartition.java | 163 +++++++
.../lens/cube/metadata/MetastoreConstants.java | 2 +
.../lens/cube/metadata/MetastoreUtil.java | 22 +-
.../org/apache/lens/cube/metadata/Storage.java | 61 +--
.../lens/cube/metadata/StorageConstants.java | 4 +
.../lens/cube/metadata/TimePartition.java | 126 +++++
.../apache/lens/cube/metadata/UpdatePeriod.java | 2 +-
.../timeline/EndsAndHolesPartitionTimeline.java | 213 +++++++++
.../metadata/timeline/PartitionTimeline.java | 207 ++++++++
.../timeline/PartitionTimelineFactory.java | 59 +++
.../timeline/StoreAllPartitionTimeline.java | 111 +++++
.../cube/parse/AbridgedTimeRangeWriter.java | 75 ++-
.../lens/cube/parse/BetweenTimeRangeWriter.java | 2 +
.../apache/lens/cube/parse/FactPartition.java | 166 -------
.../lens/cube/parse/ORTimeRangeWriter.java | 2 +
.../lens/cube/parse/StorageTableResolver.java | 263 +++++-----
.../org/apache/lens/cube/parse/StorageUtil.java | 15 +
.../org/apache/lens/cube/parse/TimeRange.java | 87 ++++
.../apache/lens/cube/parse/TimeRangeUtils.java | 2 +
.../apache/lens/cube/parse/TimeRangeWriter.java | 2 +
.../cube/metadata/TestCubeMetastoreClient.java | 476 +++++++++++--------
.../lens/cube/metadata/TestTimePartition.java | 76 +++
.../apache/lens/cube/parse/CubeTestSetup.java | 121 ++++-
.../lens/cube/parse/TestCubeRewriter.java | 13 +-
.../apache/lens/cube/parse/TestStorageUtil.java | 1 +
.../lens/cube/parse/TestTimeRangeWriter.java | 1 +
.../metastore/CubeMetastoreServiceImpl.java | 37 +-
.../server/metastore/TestMetastoreService.java | 6 +-
31 files changed, 2164 insertions(+), 684 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CaseInsensitiveStringHashMap.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CaseInsensitiveStringHashMap.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CaseInsensitiveStringHashMap.java
new file mode 100644
index 0000000..e125956
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CaseInsensitiveStringHashMap.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * HashMap from String to type T where the key is case insensitive.
+ *
+ * @param <T>
+ */
+public class CaseInsensitiveStringHashMap<T> extends HashMap<String, T> {
+ @Override
+ public T get(Object key) {
+ return super.get(caseInsensitiveKey(key));
+ }
+
+ @Override
+ public T put(String key, T value) {
+ return super.put(caseInsensitiveKey(key), value);
+ }
+
+ @Override
+ public boolean containsKey(Object key) {
+ return super.containsKey(caseInsensitiveKey(key));
+ }
+
+ private static String caseInsensitiveKey(Object key) {
+ return key == null ? null : key.toString().toLowerCase();
+ }
+
+ @Override
+ public void putAll(Map<? extends String, ? extends T> m) {
+ for (Map.Entry<? extends String, ? extends T> entry : m.entrySet()) {
+ put(entry.getKey(), entry.getValue());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
index b3ed6ef..e0e87ab 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
@@ -179,7 +179,8 @@ public final class CubeFactTable extends AbstractCubeTable {
// between the dates
UpdatePeriodComparator cmp = new UpdatePeriodComparator();
for (UpdatePeriod i : updatePeriods) {
- if (UpdatePeriod.YEARLY == i || UpdatePeriod.QUARTERLY == i || UpdatePeriod.MONTHLY == i) {
+ if (UpdatePeriod.YEARLY == i || UpdatePeriod.QUARTERLY == i || UpdatePeriod.MONTHLY == i
+ || UpdatePeriod.WEEKLY == i) {
int intervals = 0;
switch (i) {
case YEARLY:
@@ -226,8 +227,7 @@ public final class CubeFactTable extends AbstractCubeTable {
}
/**
- * Return valid columns of the fact, which can be specified by the property
- * MetastoreUtil.getValidColumnsKey(getName())
+ * Return valid columns of the fact, which can be specified by property MetastoreUtil.getValidColumnsKey(getName())
*
* @return
*/
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 7e6fe7e..2e03df1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -22,16 +22,17 @@ package org.apache.lens.cube.metadata;
import java.text.ParseException;
import java.util.*;
+import org.apache.lens.api.LensException;
import org.apache.lens.cube.metadata.Storage.LatestInfo;
import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo;
+import org.apache.lens.cube.metadata.timeline.PartitionTimeline;
+import org.apache.lens.cube.metadata.timeline.PartitionTimelineFactory;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
@@ -39,9 +40,13 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.thrift.TException;
+import com.google.common.collect.Maps;
+import lombok.extern.apachecommons.CommonsLog;
+
/**
* Wrapper class around Hive metastore to do cube metastore operations.
*/
+@CommonsLog
public class CubeMetastoreClient {
private final HiveConf config;
private final boolean enableCaching;
@@ -52,24 +57,268 @@ public class CubeMetastoreClient {
}
// map from table name to Table
- private final Map<String, Table> allHiveTables = new HashMap<String, Table>();
+ private final Map<String, Table> allHiveTables = Maps.newHashMap();
// map from dimension name to Dimension
- private final Map<String, Dimension> allDims = new HashMap<String, Dimension>();
+ private final Map<String, Dimension> allDims = Maps.newHashMap();
// map from cube name to Cube
- private final Map<String, CubeInterface> allCubes = new HashMap<String, CubeInterface>();
+ private final Map<String, CubeInterface> allCubes = Maps.newHashMap();
// map from dimtable name to CubeDimensionTable
- private final Map<String, CubeDimensionTable> allDimTables = new HashMap<String, CubeDimensionTable>();
+ private final Map<String, CubeDimensionTable> allDimTables = Maps.newHashMap();
// map from fact name to fact table
- private final Map<String, CubeFactTable> allFactTables = new HashMap<String, CubeFactTable>();
+ private final Map<String, CubeFactTable> allFactTables = Maps.newHashMap();
// map from storage name to storage
- private final Map<String, Storage> allStorages = new HashMap<String, Storage>();
-
+ private final Map<String, Storage> allStorages = Maps.newHashMap();
+ // Partition cache. Inner class since it logically belongs here
+ PartitionTimelineCache partitionTimelineCache = new PartitionTimelineCache();
// dbname to client mapping
- private static final Map<String, CubeMetastoreClient> CLIENT_MAPPING =
- new HashMap<String, CubeMetastoreClient>();
-
+ private static final Map<String, CubeMetastoreClient> CLIENT_MAPPING = Maps.newHashMap();
private SchemaGraph schemaGraph;
+ /** extract storage name from fact and storage table name. String operation */
+ private String extractStorageName(CubeFactTable fact, String storageTableName) throws LensException {
+ int ind = storageTableName.lastIndexOf(fact.getName());
+ if (ind <= 0) {
+ throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName());
+ }
+ return storageTableName.substring(0, ind - StorageConstants.STORGAE_SEPARATOR.length());
+ }
+
+ /**
+ * get latest date for timeDimension from all fact-storage tables belonging to the given cube having timeDimension,
+ * return the most recent of all.
+ * <p/>
+ * latest date for a single fact-storage table for given time dimension is the latest of the latest dates for all its
+ * update periods
+ *
+ * @param cube
+ * @param timeDimension
+ * @return
+ * @throws HiveException
+ * @throws LensException
+ */
+ public Date getLatestDateOfCube(Cube cube, String timeDimension) throws HiveException, LensException {
+ String partCol = cube.getPartitionColumnOfTimeDim(timeDimension);
+ Date max = new Date(Long.MIN_VALUE);
+ boolean updated = false;
+ for (CubeFactTable fact : getAllFactTables(cube)) {
+ for (String storage : fact.getStorages()) {
+ for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) {
+ PartitionTimeline timeline = partitionTimelineCache.get(fact.getName(), storage, updatePeriod,
+ partCol);
+ if (timeline != null) {// this storage table is partitioned by partCol or not.
+ Date latest = timeline.getLatestDate();
+ if (latest != null && latest.after(max)) {
+ max = latest;
+ updated = true;
+ }
+ }
+ }
+ }
+ }
+ return updated ? max : null;
+ }
+
+ /** clear hive table cache */
+ public void clearHiveTableCache() {
+ allHiveTables.clear();
+ }
+
+ /**
+ * In-memory storage of {@link org.apache.lens.cube.metadata.timeline.PartitionTimeline} objects for each valid
+ * storagetable-updateperiod-partitioncolumn tuple. also simultaneously stored in metastore table of the
+ * storagetable.
+ */
+ class PartitionTimelineCache extends CaseInsensitiveStringHashMap<// storage table
+ TreeMap<UpdatePeriod,
+ CaseInsensitiveStringHashMap<// partition column
+ PartitionTimeline>>> {
+ /**
+ * Returns true if all the timelines for fact-storage table are empty for all valid update periods.
+ *
+ * @param fact
+ * @param storage
+ * @param partCol
+ * @return
+ * @throws HiveException
+ * @throws LensException
+ */
+ public boolean noPartitionsExist(String fact, String storage, String partCol)
+ throws HiveException, LensException {
+ if (get(fact, storage) == null) {
+ return true;
+ }
+ for (UpdatePeriod updatePeriod : get(fact, storage).keySet()) {
+ PartitionTimeline timeline = get(fact, storage, updatePeriod, partCol);
+ if (timeline != null && !timeline.isEmpty()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * get all timelines for all update periods and partition columns for the given fact-storage pair. If already loaded
+ * in memory, it'll return that. If not, it'll first try to load it from table properties. If not found in table
+ * properties, it'll get all partitions, compute timelines in memory, write back all loads timelines to table
+ * properties for further usage and return them.
+ *
+ * @param fact
+ * @param storage
+ * @return
+ * @throws HiveException
+ * @throws LensException
+ */
+ public TreeMap<UpdatePeriod, CaseInsensitiveStringHashMap<PartitionTimeline>> get(String fact, String storage)
+ throws HiveException, LensException {
+ String storageTableName = MetastoreUtil.getStorageTableName(fact, Storage.getPrefix(storage));
+ if (get(storageTableName) == null) {
+ // not found in memory, try loading from table properties.
+ Table storageTable = getTable(storageTableName);
+ if (storageTable.getParameters().get(MetastoreUtil.getPartitoinTimelineCachePresenceKey()) == null) {
+ // Not found in table properties either, compute from all partitions of the fact-storage table.
+ // First make sure all combinations of update period and partition column have an entry even
+ // if no partitions exist
+ if (getCubeFact(fact).getUpdatePeriods() != null && getCubeFact(fact).getUpdatePeriods().get(
+ storage) != null) {
+ for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
+ for (String partCol : getTimePartsOfTable(storageTable)) {
+ partitionTimelineCache.ensureEntry(storageTableName, updatePeriod, partCol);
+ }
+ }
+ }
+ // Then add all existing partitions for batch addition in respective timelines.
+ List<String> timeParts = getTimePartsOfTable(storageTable);
+ List<FieldSchema> partCols = storageTable.getPartCols();
+ for (Partition partition : getPartitionsByFilter(storageTableName, null)) {
+ UpdatePeriod period = deduceUpdatePeriod(partition);
+ List<String> values = partition.getValues();
+ for (int i = 0; i < partCols.size(); i++) {
+ if (timeParts.contains(partCols.get(i).getName())) {
+ partitionTimelineCache.addForBatchAddition(storageTableName, period, partCols.get(i).getName(),
+ values.get(i));
+ }
+ }
+ }
+ // commit all batch addition for the storage table, which will in-turn commit all batch additions in all it's
+ // timelines.
+ commitAllBatchAdditions(storageTableName);
+ } else {
+ // found in table properties, load from there.
+ for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
+ for (String partCol : getTimePartsOfTable(storageTableName)) {
+ ensureEntry(storageTableName, updatePeriod, partCol).init(storageTable);
+ }
+ }
+ }
+ }
+ // return the final value from memory
+ return get(storageTableName);
+ }
+
+ /**
+ * Adds given partition(for storageTable, updatePeriod, partitionColum=partition) for batch addition in an
+ * appropriate timeline object. Ignore if partition is not valid.
+ *
+ * @param storageTable
+ * @param updatePeriod
+ * @param partitionColumn
+ * @param partition
+ */
+ public void addForBatchAddition(String storageTable, UpdatePeriod updatePeriod, String partitionColumn,
+ String partition) {
+ try {
+ ensureEntry(storageTable, updatePeriod, partitionColumn).addForBatchAddition(TimePartition.of(updatePeriod,
+ partition));
+ } catch (LensException e) {
+ // to take care of the case where partition name is something like `latest`
+ log.error("Couldn't parse partition: " + partition + " with update period: " + updatePeriod + ", skipping.", e);
+ }
+ }
+
+ /**
+ * helper method for ensuring get(storageTable).get(updatePeriod).get(partitionColumn) gives a non-null object.
+ * <p/>
+ * kind of like mkdir -p
+ *
+ * @param storageTable
+ * @param updatePeriod
+ * @param partitionColumn
+ * @return timeline if already exists, or puts a new timeline and returns.
+ */
+ public PartitionTimeline ensureEntry(String storageTable, UpdatePeriod updatePeriod, String partitionColumn) {
+ if (get(storageTable) == null) {
+ put(storageTable, new TreeMap<UpdatePeriod, CaseInsensitiveStringHashMap<PartitionTimeline>>());
+ }
+ if (get(storageTable).get(updatePeriod) == null) {
+ get(storageTable).put(updatePeriod, new CaseInsensitiveStringHashMap<PartitionTimeline>());
+ }
+ if (get(storageTable).get(updatePeriod).get(partitionColumn) == null) {
+ get(storageTable).get(updatePeriod).put(partitionColumn, PartitionTimelineFactory.get(
+ CubeMetastoreClient.this, storageTable, updatePeriod, partitionColumn));
+ }
+ return get(storageTable).get(updatePeriod).get(partitionColumn);
+ }
+
+ /**
+ * commit all batch addition for all its timelines.
+ *
+ * @param storageTable
+ * @throws HiveException
+ * @throws LensException
+ */
+ public void commitAllBatchAdditions(String storageTable) throws HiveException, LensException {
+ if (get(storageTable) != null) {
+ for (UpdatePeriod updatePeriod : get(storageTable).keySet()) {
+ for (String partCol : get(storageTable).get(updatePeriod).keySet()) {
+ PartitionTimeline timeline = get(storageTable).get(updatePeriod).get(partCol);
+ timeline.commitBatchAdditions();
+ }
+ }
+ alterTablePartitionCache(storageTable);
+ }
+ }
+
+ /** check partition existence in the appropriate timeline if it exists */
+ public boolean partitionTimeExists(String name, String storage, UpdatePeriod period, String partCol, Date partSpec)
+ throws HiveException, LensException {
+ return get(name, storage, period, partCol) != null && get(name, storage, period, partCol).exists(TimePartition.of(
+ period, partSpec));
+ }
+
+ /**
+ * returns the timeline corresponding to fact-storage table, updatePeriod, partCol. null if doesn't exist, which
+ * would only happen if the combination is not valid/supported
+ */
+ public PartitionTimeline get(String fact, String storage, UpdatePeriod updatePeriod, String partCol)
+ throws HiveException, LensException {
+ return get(fact, storage) != null && get(fact, storage).get(updatePeriod) != null && get(fact, storage).get(
+ updatePeriod).get(
+ partCol) != null ? get(fact, storage).get(updatePeriod).get(partCol) : null;
+ }
+
+ /** update partition timeline cache for addition of time partition */
+ public void updateForAddition(String cubeTableName, String storageName, UpdatePeriod updatePeriod,
+ Map<String, Date> timePartSpec)
+ throws HiveException, LensException {
+ for (Map.Entry<String, Date> entry : timePartSpec.entrySet()) {
+ //Assume timelines has all the time part columns.
+ get(cubeTableName, storageName, updatePeriod, entry.getKey()).add(TimePartition.of(updatePeriod,
+ entry.getValue()));
+ }
+ }
+
+ /** update partition timeline cache for deletion of time partition */
+ public void updateForDeletion(String cubeTableName, String storageName, UpdatePeriod updatePeriod,
+ Map<String, Date> timePartSpec) throws HiveException, LensException {
+ for (Map.Entry<String, Date> entry : timePartSpec.entrySet()) {
+ get(cubeTableName, storageName, updatePeriod, entry.getKey()).drop(TimePartition.of(
+ updatePeriod, entry.getValue()));
+ }
+ }
+
+ }
+
+
/**
* Get the instance of {@link CubeMetastoreClient} corresponding to {@link HiveConf}
*
@@ -359,20 +608,62 @@ public class CubeMetastoreClient {
/**
* Add a partition specified by the storage partition desc on the storage passed.
+ * <p/>
+ * TODO: separate methods for fact and dim partition addition.
*
* @param partSpec The storage partition description
* @param storageName The storage object
* @throws HiveException
*/
- public void addPartition(StoragePartitionDesc partSpec, String storageName) throws HiveException {
- String storageTableName =
- MetastoreUtil.getStorageTableName(partSpec.getCubeTableName(), Storage.getPrefix(storageName));
+ public void addPartition(StoragePartitionDesc partSpec, String storageName) throws HiveException, LensException {
+ String storageTableName = MetastoreUtil.getStorageTableName(partSpec.getCubeTableName(), Storage.getPrefix(
+ storageName));
+ if (getDimensionTable(partSpec.getCubeTableName()) != null) {
+ // Adding partition in dimension table.
+ getStorage(storageName).addPartition(getClient(), partSpec,
+ getDimTableLatestInfo(storageTableName, partSpec.getTimePartSpec(), partSpec.getUpdatePeriod())
+ );
+ } else {
+ // first update in memory, then add to hive table's partitions. delete is reverse.
+ partitionTimelineCache.updateForAddition(partSpec.getCubeTableName(), storageName, partSpec.getUpdatePeriod(),
+ partSpec.getTimePartSpec());
+ // Adding partition in fact table.
+ getStorage(storageName).addPartition(getClient(), partSpec, null);
+ // update hive table
+ alterTablePartitionCache(MetastoreUtil.getStorageTableName(partSpec.getCubeTableName(), Storage.getPrefix(
+ storageName)));
+ }
+ }
- getStorage(storageName).addPartition(getClient(), partSpec,
- getLatestInfo(storageTableName, partSpec.getTimePartSpec(), partSpec.getUpdatePeriod()));
+ /**
+ * store back all timelines of given storage table to table properties
+ *
+ * @param storageTableName
+ * @throws HiveException
+ */
+ private void alterTablePartitionCache(String storageTableName) throws HiveException {
+ Table table = getTable(storageTableName);
+ Map<String, String> params = table.getParameters();
+ if (partitionTimelineCache.get(storageTableName) != null) {
+ for (UpdatePeriod updatePeriod : partitionTimelineCache.get(storageTableName).keySet()) {
+ for (Map.Entry<String, PartitionTimeline> entry : partitionTimelineCache.get(storageTableName)
+ .get(updatePeriod).entrySet()) {
+ entry.getValue().updateTableParams(table);
+ }
+ }
+ params.put(MetastoreUtil.getPartitoinTimelineCachePresenceKey(), "true");
+ alterHiveTable(storageTableName, table);
+ }
+ }
+
+ /** extract update period from partition properties */
+ private UpdatePeriod deduceUpdatePeriod(Partition partition) {
+ return UpdatePeriod.valueOf(partition.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD));
}
- public void addPartitions(List<StoragePartitionDesc> storagePartitionDescs, String storageName) throws HiveException {
+ /** batch addition */
+ public void addPartitions(List<StoragePartitionDesc> storagePartitionDescs, String storageName)
+ throws HiveException, LensException {
//TODO: improve this in later jira. Just providing naive implementation for now.
// Should ideally do some optimization because the list has been provided together, not one by one.
for (StoragePartitionDesc partSpec : storagePartitionDescs) {
@@ -380,7 +671,7 @@ public class CubeMetastoreClient {
}
}
- private LatestInfo getLatestInfo(String storageTableName, Map<String, Date> partitionTimestamps,
+ private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, Date> partitionTimestamps,
UpdatePeriod updatePeriod) throws HiveException {
Table hiveTable = getHiveTable(storageTableName);
String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS);
@@ -394,7 +685,7 @@ public class CubeMetastoreClient {
}
boolean makeLatest = true;
Partition part = getLatestPart(storageTableName, partCol);
- Date latestTimestamp = getLatestTimeStamp(part, partCol);
+ Date latestTimestamp = getLatestTimeStampOfDimtable(part, partCol);
if (latestTimestamp != null && pTimestamp.before(latestTimestamp)) {
makeLatest = false;
}
@@ -411,11 +702,11 @@ public class CubeMetastoreClient {
}
}
- private boolean isLatestPart(Partition part) {
+ private boolean isLatestPartOfDimtable(Partition part) {
return part.getValues().contains(StorageConstants.LATEST_PARTITION_VALUE);
}
- public Date getLatestTimeStamp(Partition part, String partCol) throws HiveException {
+ public Date getLatestTimeStampOfDimtable(Partition part, String partCol) throws HiveException {
if (part != null) {
String latestTimeStampStr = part.getParameters().get(MetastoreUtil.getLatestPartTimestampKey(partCol));
String latestPartUpdatePeriod = part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD);
@@ -444,7 +735,8 @@ public class CubeMetastoreClient {
return partDate;
}
- private LatestInfo getNextLatest(Table hiveTable, String timeCol, final int timeColIndex) throws HiveException {
+ private LatestInfo getNextLatestOfDimtable(Table hiveTable, String timeCol, final int timeColIndex)
+ throws HiveException {
// getClient().getPartitionsByNames(tbl, partNames)
List<Partition> partitions = getClient().getPartitions(hiveTable);
@@ -469,7 +761,7 @@ public class CubeMetastoreClient {
}
});
for (Partition part : partitions) {
- if (!isLatestPart(part)) {
+ if (!isLatestPartOfDimtable(part)) {
Date partDate = getPartDate(part, timeColIndex);
if (partDate != null) {
allPartTimeVals.add(part);
@@ -502,7 +794,7 @@ public class CubeMetastoreClient {
* @throws HiveException
*/
public void dropPartition(String cubeTableName, String storageName, Map<String, Date> timePartSpec,
- Map<String, String> nonTimePartSpec, UpdatePeriod updatePeriod) throws HiveException {
+ Map<String, String> nonTimePartSpec, UpdatePeriod updatePeriod) throws HiveException, LensException {
String storageTableName = MetastoreUtil.getStorageTableName(cubeTableName, Storage.getPrefix(storageName));
Table hiveTable = getHiveTable(storageTableName);
List<FieldSchema> partCols = hiveTable.getPartCols();
@@ -518,47 +810,55 @@ public class CubeMetastoreClient {
throw new HiveException("Invalid partspec, missing value for" + column.getName());
}
}
- String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS);
- Map<String, LatestInfo> latest = new HashMap<String, Storage.LatestInfo>();
- if (timePartColsStr != null) {
- List<String> timePartCols = Arrays.asList(StringUtils.split(timePartColsStr, ','));
- for (String timeCol : timePartSpec.keySet()) {
- if (!timePartCols.contains(timeCol)) {
- throw new HiveException("Not a time partition column:" + timeCol);
- }
- int timeColIndex = partColNames.indexOf(timeCol);
- Partition part = getLatestPart(storageTableName, timeCol);
-
- // check if partition being dropped is the latest partition
- boolean isLatest = true;
- for (int i = 0; i < partVals.size(); i++) {
- if (i != timeColIndex) {
- if (!part.getValues().get(i).equals(partVals.get(i))) {
- isLatest = false;
- break;
- }
+ if (isDimensionTable(cubeTableName)) {
+ String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS);
+ Map<String, LatestInfo> latest = new HashMap<String, Storage.LatestInfo>();
+ if (timePartColsStr != null) {
+ List<String> timePartCols = Arrays.asList(StringUtils.split(timePartColsStr, ','));
+ for (String timeCol : timePartSpec.keySet()) {
+ if (!timePartCols.contains(timeCol)) {
+ throw new HiveException("Not a time partition column:" + timeCol);
}
- }
- if (isLatest) {
- Date latestTimestamp = getLatestTimeStamp(part, timeCol);
- Date dropTimestamp;
- try {
- dropTimestamp = updatePeriod.format().parse(updatePeriod.format().format(timePartSpec.get(timeCol)));
- } catch (ParseException e) {
- throw new HiveException(e);
+ int timeColIndex = partColNames.indexOf(timeCol);
+ Partition part = getLatestPart(storageTableName, timeCol);
+
+ // check if partition being dropped is the latest partition
+ boolean isLatest = true;
+ for (int i = 0; i < partVals.size(); i++) {
+ if (i != timeColIndex) {
+ if (!part.getValues().get(i).equals(partVals.get(i))) {
+ isLatest = false;
+ break;
+ }
+ }
}
- if (latestTimestamp != null && dropTimestamp.equals(latestTimestamp)) {
- LatestInfo latestInfo = getNextLatest(hiveTable, timeCol, timeColIndex);
- latest.put(timeCol, latestInfo);
+ if (isLatest) {
+ Date latestTimestamp = getLatestTimeStampOfDimtable(part, timeCol);
+ Date dropTimestamp;
+ try {
+ dropTimestamp = updatePeriod.format().parse(updatePeriod.format().format(timePartSpec.get(timeCol)));
+ } catch (ParseException e) {
+ throw new HiveException(e);
+ }
+ if (latestTimestamp != null && dropTimestamp.equals(latestTimestamp)) {
+ LatestInfo latestInfo = getNextLatestOfDimtable(hiveTable, timeCol, timeColIndex);
+ latest.put(timeCol, latestInfo);
+ }
}
}
+ } else {
+ if (timePartSpec != null && !timePartSpec.isEmpty()) {
+ throw new HiveException("Not time part columns" + timePartSpec.keySet());
+ }
}
+ getStorage(storageName).dropPartition(getClient(), storageTableName, partVals,
+ latest);
} else {
- if (timePartSpec != null && !timePartSpec.isEmpty()) {
- throw new HiveException("Not time part columns" + timePartSpec.keySet());
- }
+ // dropping fact partition
+ getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, null);
+ partitionTimelineCache.updateForDeletion(cubeTableName, storageName, updatePeriod, timePartSpec);
+ this.alterTablePartitionCache(storageTableName);
}
- getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, latest);
}
private Map<String, String> getPartitionSpec(UpdatePeriod updatePeriod, Map<String, Date> partitionTimestamps) {
@@ -578,7 +878,15 @@ public class CubeMetastoreClient {
}
}
- boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod,
+ /** extract storage name and check in timeline cache for existance */
+ public boolean factPartitionExists(CubeFactTable fact, FactPartition part, String storageTableName)
+ throws HiveException, LensException {
+ String storage = extractStorageName(fact, storageTableName);
+ return partitionTimelineCache.partitionTimeExists(fact.getName(), storage, part.getPeriod(), part.getPartCol(),
+ part.getPartSpec());
+ }
+
+ public boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod,
Map<String, Date> partitionTimestamp, Map<String, String> partSpec) throws HiveException {
String storageTableName = MetastoreUtil.getFactStorageTableName(factName, storageName);
return partitionExists(storageTableName, updatePeriod, partitionTimestamp, partSpec);
@@ -611,8 +919,7 @@ public class CubeMetastoreClient {
}
}
- public int getNumPartitionsByFilter(String storageTableName, String filter) throws MetaException,
- NoSuchObjectException, HiveException, TException {
+ public int getNumPartitionsByFilter(String storageTableName, String filter) throws HiveException, TException {
return getClient().getNumPartitionsByFilter(getTable(storageTableName), filter);
}
@@ -639,9 +946,14 @@ public class CubeMetastoreClient {
partitionTimestamps);
}
- boolean latestPartitionExists(String factName, String storageName, String latestPartCol) throws HiveException {
+ boolean latestPartitionExists(String factName, String storageName, String latestPartCol)
+ throws HiveException, LensException {
String storageTableName = MetastoreUtil.getFactStorageTableName(factName, storageName);
- return partitionExistsByFilter(storageTableName, StorageConstants.getLatestPartFilter(latestPartCol));
+ if (isDimensionTable(factName)) {
+ return partitionExistsByFilter(storageTableName, StorageConstants.getLatestPartFilter(latestPartCol));
+ } else {
+ return !partitionTimelineCache.noPartitionsExist(factName, storageName, latestPartCol);
+ }
}
Partition getLatestPart(String storageTableName, String latestPartCol) throws HiveException {
@@ -664,7 +976,18 @@ public class CubeMetastoreClient {
return getTable(tableName);
}
- private Table getTable(String tableName) throws HiveException {
+ private List<String> getTimePartsOfTable(String storageTableName) throws HiveException {
+ return getTimePartsOfTable(getTable(storageTableName));
+ }
+
+ /** extract from table properties */
+ public List<String> getTimePartsOfTable(Table table) {
+ List<String> ret = Arrays.asList(StringUtils.split(table.getParameters().get(MetastoreConstants.TIME_PART_COLUMNS),
+ ","));
+ return ret == null ? new ArrayList<String>() : ret;
+ }
+
+ public Table getTable(String tableName) throws HiveException {
Table tbl;
try {
tbl = allHiveTables.get(tableName.toLowerCase());
@@ -818,8 +1141,7 @@ public class CubeMetastoreClient {
*/
public CubeFactTable getFactTable(String tableName) throws HiveException {
- Table tbl = getTable(tableName);
- return getFactTable(tbl);
+ return getFactTable(getTable(tableName));
}
private CubeFactTable getFactTable(Table tbl) throws HiveException {
@@ -928,8 +1250,7 @@ public class CubeMetastoreClient {
public CubeFactTable getCubeFact(String tableName) throws HiveException {
CubeFactTable fact = allFactTables.get(tableName.toLowerCase());
if (fact == null) {
- Table tbl = getTable(tableName);
- fact = getFactTable(tbl);
+ fact = getFactTable(tableName);
if (enableCaching) {
allFactTables.put(tableName.toLowerCase(), fact);
}
@@ -1198,8 +1519,11 @@ public class CubeMetastoreClient {
return columnsChanged;
}
- private void alterHiveTable(String table, Table hiveTable, List<FieldSchema> columns) throws HiveException {
- hiveTable.getTTable().getSd().setCols(columns);
+ public void pushHiveTable(Table hiveTable) throws HiveException {
+ alterHiveTable(hiveTable.getTableName(), hiveTable);
+ }
+
+ public void alterHiveTable(String table, Table hiveTable) throws HiveException {
try {
getClient().alterTable(table, hiveTable);
} catch (InvalidOperationException e) {
@@ -1207,10 +1531,16 @@ public class CubeMetastoreClient {
}
if (enableCaching) {
// refresh the table in cache
+ //TODO: don't need to fetch again. can put table -> hiveTable
refreshTable(table);
}
}
+ private void alterHiveTable(String table, Table hiveTable, List<FieldSchema> columns) throws HiveException {
+ hiveTable.getTTable().getSd().setCols(columns);
+ alterHiveTable(table, hiveTable);
+ }
+
/**
* Alter cube specified by the name to new definition
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
new file mode 100644
index 0000000..729da30
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.cube.metadata;
+
+import java.text.DateFormat;
+import java.util.Date;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
+
+@EqualsAndHashCode
+public class FactPartition implements Comparable<FactPartition> {
+ @Getter
+ private final String partCol;
+ @Getter
+ private final Date partSpec;
+ @Getter
+ private final Set<String> storageTables = new LinkedHashSet<String>();
+ @Getter
+ private final UpdatePeriod period;
+ @Getter
+ @Setter
+ private FactPartition containingPart;
+ private final DateFormat partFormat;
+
+ public FactPartition(String partCol, Date partSpec, UpdatePeriod period,
+ FactPartition containingPart, DateFormat partFormat) {
+ this.partCol = partCol;
+ this.partSpec = partSpec;
+ this.period = period;
+ this.containingPart = containingPart;
+ this.partFormat = partFormat;
+ }
+
+ public FactPartition(String partCol, Date partSpec, UpdatePeriod period, FactPartition containingPart,
+ DateFormat partFormat, Set<String> storageTables) {
+ this(partCol, partSpec, period, containingPart, partFormat);
+ this.storageTables.addAll(storageTables);
+ }
+
+ public boolean hasContainingPart() {
+ return containingPart != null;
+ }
+
+ public boolean found() {
+ return !getStorageTables().isEmpty();
+ }
+
+ public String getFormattedPartSpec() {
+ if (partFormat == null) {
+ return getPartString();
+ } else {
+ return partFormat.format(partSpec);
+ }
+ }
+
+ public String getPartString() {
+ return period.format().format(partSpec);
+ }
+
+ public String getFormattedFilter(String tableName) {
+ return getFormattedFilter(partCol, tableName);
+ }
+
+ public String getFormattedFilter(String partCol, String tableName) {
+ StringBuilder builder = new StringBuilder();
+ if (containingPart != null) {
+ builder.append(containingPart.getFormattedFilter(tableName));
+ builder.append(" AND ");
+ }
+ if (tableName != null) {
+ builder.append(tableName);
+ builder.append(".");
+ }
+ builder.append(partCol);
+ builder.append(" = '").append(getFormattedPartSpec()).append("'");
+ return builder.toString();
+ }
+
+ public String getFilter() {
+ StringBuilder builder = new StringBuilder();
+ if (containingPart != null) {
+ builder.append(containingPart.getFilter());
+ builder.append(" AND ");
+ }
+ builder.append(partCol);
+ builder.append(" = '").append(getPartString()).append("'");
+ return builder.toString();
+ }
+
+ @Override
+ public String toString() {
+ return getFilter();
+ }
+
+ public int compareTo(FactPartition o) {
+ int colComp = this.partCol.compareTo(o.partCol);
+ if (colComp == 0) {
+ int partComp = 0;
+ if (this.partSpec != null) {
+ if (o.partSpec == null) {
+ partComp = 1;
+ }
+ partComp = this.partSpec.compareTo(o.partSpec);
+ } else {
+ if (o.partSpec != null) {
+ partComp = -1;
+ } else {
+ partComp = 0;
+ }
+ }
+ if (partComp == 0) {
+ int upComp = 0;
+ if (this.period != null && o.period != null) {
+ upComp = this.period.compareTo(o.period);
+ } else if (this.period == null && o.period == null) {
+ upComp = 0;
+ } else if (this.period == null) {
+ upComp = -1;
+ } else {
+ upComp = 1;
+ }
+ if (upComp == 0) {
+ if (this.containingPart != null) {
+ if (o.containingPart == null) {
+ return 1;
+ }
+ return this.containingPart.compareTo(o.containingPart);
+ } else {
+ if (o.containingPart != null) {
+ return -1;
+ } else {
+ return 0;
+ }
+ }
+ }
+ return upComp;
+ }
+ return partComp;
+ }
+ return colComp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
index 979de6d..7e6360e 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
@@ -97,6 +97,8 @@ public final class MetastoreConstants {
// storage constants
public static final String STORAGE_ENTITY_PFX = "storage.";
public static final String STORAGE_PFX = "cube.storagetable.";
+ public static final String PARTITION_TIMELINE_CACHE = "partition.timeline.cache.";
+ public static final String STORAGE_CLASS = "storage.class";
public static final String TIME_PART_COLUMNS = "cube.storagetable.time.partcols";
public static final String LATEST_PART_TIMESTAMP_SFX = ".latest.part.timestamp";
public static final String PARTITION_UPDATE_PERIOD_SFX = ".partition.update.period";
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index c21c72a..0be8e5f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -347,16 +347,16 @@ public class MetastoreUtil {
// Utils ///
// /////////////////////////
public static <E extends Named> String getNamedStr(Collection<E> set) {
- if (set == null || set.isEmpty()) {
+ if (set == null) {
return "";
}
+ String sep = "";
StringBuilder valueStr = new StringBuilder();
Iterator<E> it = set.iterator();
- for (int i = 0; i < (set.size() - 1); i++) {
- valueStr.append(it.next().getName());
- valueStr.append(",");
+ while (it.hasNext()) {
+ valueStr.append(sep).append(it.next().getName());
+ sep = ",";
}
- valueStr.append(it.next().getName());
return valueStr.toString();
}
@@ -457,4 +457,16 @@ public class MetastoreUtil {
cols.add(dim.getName().toLowerCase());
}
}
+
+ public static String getPartitionInfoKeyPrefix(UpdatePeriod updatePeriod, String partCol) {
+ return STORAGE_PFX + PARTITION_TIMELINE_CACHE + updatePeriod.getName() + "." + partCol + ".";
+ }
+
+ public static String getPartitionTimelineStorageClassKey(UpdatePeriod updatePeriod, String partCol) {
+ return getPartitionInfoKeyPrefix(updatePeriod, partCol) + STORAGE_CLASS;
+ }
+
+ public static String getPartitoinTimelineCachePresenceKey() {
+ return STORAGE_PFX + PARTITION_TIMELINE_CACHE + "present";
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
index 240516d..f1692b2 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
@@ -326,35 +326,38 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
String dbName = SessionState.get().getCurrentDatabase();
Table storageTbl = client.getTable(storageTableName);
// update latest info
- for (Map.Entry<String, LatestInfo> entry : updateLatestInfo.entrySet()) {
- String latestPartCol = entry.getKey();
- // symlink this partition to latest
- List<Partition> latestParts;
- try {
- latestParts = client.getPartitionsByFilter(storageTbl, StorageConstants.getLatestPartFilter(latestPartCol));
- } catch (Exception e) {
- throw new HiveException("Could not get latest partition", e);
- }
- if (!latestParts.isEmpty()) {
- client.dropPartition(storageTbl.getTableName(), latestParts.get(0).getValues(), false);
- }
- LatestInfo latest = entry.getValue();
- if (latest != null && latest.part != null) {
- AddPartitionDesc latestPart = new AddPartitionDesc(dbName, storageTableName, true);
- latestPart.addPartition(StorageConstants.getLatestPartSpec(latest.part.getSpec(), latestPartCol),
- latest.part.getLocation());
- latestPart.getPartition(0).setPartParams(
- latest.latestParts.get(latestPartCol).getPartParams(latest.part.getParameters()));
- latestPart.getPartition(0).setInputFormat(latest.part.getInputFormatClass().getCanonicalName());
- latestPart.getPartition(0).setOutputFormat(latest.part.getOutputFormatClass().getCanonicalName());
- latestPart.getPartition(0).setNumBuckets(latest.part.getBucketCount());
- latestPart.getPartition(0).setCols(latest.part.getCols());
- latestPart.getPartition(0).setSerializationLib(
- latest.part.getTPartition().getSd().getSerdeInfo().getSerializationLib());
- latestPart.getPartition(0).setSerdeParams(latest.part.getTPartition().getSd().getSerdeInfo().getParameters());
- latestPart.getPartition(0).setBucketCols(latest.part.getBucketCols());
- latestPart.getPartition(0).setSortCols(latest.part.getSortCols());
- client.createPartitions(latestPart);
+ if (updateLatestInfo != null) {
+ for (Map.Entry<String, LatestInfo> entry : updateLatestInfo.entrySet()) {
+ String latestPartCol = entry.getKey();
+ // symlink this partition to latest
+ List<Partition> latestParts;
+ try {
+ latestParts = client.getPartitionsByFilter(storageTbl, StorageConstants.getLatestPartFilter(latestPartCol));
+ } catch (Exception e) {
+ throw new HiveException("Could not get latest partition", e);
+ }
+ if (!latestParts.isEmpty()) {
+ client.dropPartition(storageTbl.getTableName(), latestParts.get(0).getValues(), false);
+ }
+ LatestInfo latest = entry.getValue();
+ if (latest != null && latest.part != null) {
+ AddPartitionDesc latestPart = new AddPartitionDesc(dbName, storageTableName, true);
+ latestPart.addPartition(StorageConstants.getLatestPartSpec(latest.part.getSpec(), latestPartCol),
+ latest.part.getLocation());
+ latestPart.getPartition(0).setPartParams(
+ latest.latestParts.get(latestPartCol).getPartParams(latest.part.getParameters()));
+ latestPart.getPartition(0).setInputFormat(latest.part.getInputFormatClass().getCanonicalName());
+ latestPart.getPartition(0).setOutputFormat(latest.part.getOutputFormatClass().getCanonicalName());
+ latestPart.getPartition(0).setNumBuckets(latest.part.getBucketCount());
+ latestPart.getPartition(0).setCols(latest.part.getCols());
+ latestPart.getPartition(0).setSerializationLib(
+ latest.part.getTPartition().getSd().getSerdeInfo().getSerializationLib());
+ latestPart.getPartition(0).setSerdeParams(
+ latest.part.getTPartition().getSd().getSerdeInfo().getParameters());
+ latestPart.getPartition(0).setBucketCols(latest.part.getBucketCols());
+ latestPart.getPartition(0).setSortCols(latest.part.getSortCols());
+ client.createPartitions(latestPart);
+ }
}
}
commitDropPartition(storageTableName, partVals);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
index c4b3c3a..aee9f7b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
@@ -43,6 +43,10 @@ public final class StorageConstants {
return partCol + "='" + LATEST_PARTITION_VALUE + "'";
}
+ public static String getPartFilter(String partCol, String value) {
+ return partCol + "='" + value + "'";
+ }
+
/**
* Get the latest partition value as List
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
new file mode 100644
index 0000000..6a5b31d
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import java.text.ParseException;
+import java.util.Calendar;
+import java.util.Date;
+
+import org.apache.lens.api.LensException;
+
+import org.apache.commons.lang3.time.DateUtils;
+
+import lombok.Data;
+import lombok.NonNull;
+
+/** stores a partition's update period, date and string representation. Provides some utility methods around it */
+@Data
+public class TimePartition implements Comparable<TimePartition> {
+ private static final String UPDATE_PERIOD_WRONG_ERROR_MESSAGE = "Update period %s not correct for parsing %s";
+ private final UpdatePeriod updatePeriod;
+ private final Date date;
+ private final String dateString;
+
+ private TimePartition(@NonNull UpdatePeriod updatePeriod, @NonNull Date date) {
+ this.updatePeriod = updatePeriod;
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(date);
+ this.date = truncate(date, updatePeriod);
+ this.dateString = updatePeriod.format().format(this.date);
+ }
+
+ private TimePartition(@NonNull UpdatePeriod updatePeriod, @NonNull String dateString) throws LensException {
+ this.updatePeriod = updatePeriod;
+ this.dateString = dateString;
+ try {
+ this.date = updatePeriod.format().parse(dateString);
+ } catch (ParseException e) {
+ throw new LensException(getWrongUpdatePeriodMessage(updatePeriod, dateString), e);
+ }
+ if (!updatePeriod.format().format(this.date).equals(this.dateString)) {
+ throw new LensException(getWrongUpdatePeriodMessage(updatePeriod, dateString));
+ }
+ }
+
+ public static TimePartition of(UpdatePeriod updatePeriod, Date date) throws LensException {
+ if (date == null) {
+ throw new LensException("time parition date is null");
+ }
+ return new TimePartition(updatePeriod, date);
+ }
+
+ public static TimePartition of(UpdatePeriod updatePeriod, String dateString) throws LensException {
+ if (dateString == null || dateString.isEmpty()) {
+ throw new LensException("time parition date string is null or blank");
+ } else {
+ return new TimePartition(updatePeriod, dateString);
+ }
+ }
+
+ public String toString() {
+ return dateString;
+ }
+
+ @Override
+ public int compareTo(TimePartition o) {
+ if (o == null) {
+ return 1;
+ }
+ return this.date.compareTo(o.date);
+ }
+
+ public TimePartition partitionAtDiff(int increment) {
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(date);
+ cal.add(updatePeriod.calendarField(), increment);
+ return new TimePartition(updatePeriod, cal.getTime());
+ }
+
+ public TimePartition previous() {
+ return partitionAtDiff(-1);
+ }
+
+ public TimePartition next() {
+ return partitionAtDiff(1);
+ }
+
+ public boolean before(TimePartition when) {
+ return this.date.before(when.date);
+ }
+
+ public boolean after(TimePartition when) {
+ return this.date.after(when.date);
+ }
+
+ private Date truncate(Date date, UpdatePeriod updatePeriod) {
+ if (updatePeriod.equals(UpdatePeriod.WEEKLY)) {
+ Date truncDate = DateUtils.truncate(date, Calendar.DAY_OF_MONTH);
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(truncDate);
+ cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
+ return cal.getTime();
+ } else {
+ return DateUtils.truncate(date, updatePeriod.calendarField());
+ }
+ }
+
+ protected static String getWrongUpdatePeriodMessage(UpdatePeriod up, String dateString) {
+ return String.format(UPDATE_PERIOD_WRONG_ERROR_MESSAGE, up, dateString);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
index f48acf2..e349f58 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
@@ -31,7 +31,7 @@ public enum UpdatePeriod implements Named {
SECONDLY(Calendar.SECOND, 1000, "yyyy-MM-dd-HH-mm-ss"), MINUTELY(Calendar.MINUTE, 60 * SECONDLY.weight(),
"yyyy-MM-dd-HH-mm"), HOURLY(Calendar.HOUR_OF_DAY, 60 * MINUTELY.weight(), "yyyy-MM-dd-HH"), DAILY(
Calendar.DAY_OF_MONTH, 24 * HOURLY.weight(), "yyyy-MM-dd"), WEEKLY(Calendar.WEEK_OF_YEAR, 7 * DAILY.weight(),
- "YYYY-'W'ww"), MONTHLY(Calendar.MONTH, 30 * DAILY.weight(), "yyyy-MM"), QUARTERLY(Calendar.MONTH, 3 * MONTHLY
+ "yyyy-'W'ww"), MONTHLY(Calendar.MONTH, 30 * DAILY.weight(), "yyyy-MM"), QUARTERLY(Calendar.MONTH, 3 * MONTHLY
.weight(), "yyyy-MM"), YEARLY(Calendar.YEAR, 12 * MONTHLY.weight(), "yyyy");
public static final long MIN_INTERVAL = SECONDLY.weight();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
new file mode 100644
index 0000000..57dae9e
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata.timeline;
+
+
+import java.util.*;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.cube.metadata.CubeMetastoreClient;
+import org.apache.lens.cube.metadata.TimePartition;
+import org.apache.lens.cube.metadata.UpdatePeriod;
+import org.apache.lens.cube.parse.TimeRange;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import lombok.Data;
+import lombok.NonNull;
+
+/**
+ * One implementation of PartitionTimeline that stores first partition, latest partition and a collection of holes in
+ * between them, excluding the edges(start and end values). This is the default Timeline for tables that don't specify
+ * which Timeline Class to use in it's params.
+ */
+@Data
+public class EndsAndHolesPartitionTimeline extends PartitionTimeline {
+ private TimePartition first;
+ private TreeSet<TimePartition> holes = new TreeSet<TimePartition>();
+ private TimePartition latest;
+
+ public EndsAndHolesPartitionTimeline(CubeMetastoreClient client, String storageTableName, UpdatePeriod updatePeriod,
+ String partCol) {
+ super(client, storageTableName, updatePeriod, partCol);
+ }
+
+ @Override
+ public boolean add(TimePartition partition) throws LensException {
+ if (isEmpty()) {
+ // First partition being added
+ first = partition;
+ latest = partition;
+ return true;
+ }
+ if (partition.before(first)) {
+ addHolesBetween(partition, first, partition.getUpdatePeriod());
+ first = partition;
+ return true;
+ } else if (partition.after(latest)) {
+ addHolesBetween(latest, partition, partition.getUpdatePeriod());
+ latest = partition;
+ return true;
+ } else {
+ return holes.remove(partition);
+ }
+ }
+
+ @Override
+ public boolean add(@NonNull Collection<TimePartition> partitions) throws LensException {
+ boolean result = true;
+ for (TimePartition partition : partitions) {
+ result &= add(partition);
+ }
+ // Can also return the failed to add items.
+ return result;
+ }
+
+ @Override
+ public boolean drop(TimePartition toDrop) throws LensException {
+ if (morePartitionsExist(toDrop.getDateString())) {
+ return true;
+ }
+ if (first.equals(latest) && first.equals(toDrop)) {
+ this.first = null;
+ this.latest = null;
+ this.holes.clear();
+ return true;
+ } else if (first.equals(toDrop)) {
+ this.first = this.getNextPartition(first, latest, 1);
+ return true;
+ } else if (latest.equals(toDrop)) {
+ this.latest = this.getNextPartition(latest, first, -1);
+ return true;
+ } else {
+ return addHole(toDrop);
+ }
+ }
+
+ @Override
+ public TimePartition latest() {
+ return latest;
+ }
+
+ @Override
+ public Map<String, String> toProperties() {
+ HashMap<String, String> ret = Maps.newHashMap();
+ if (isEmpty()) {
+ return ret;
+ }
+ ret.put("first", first.getDateString());
+ ret.put("latest", latest.getDateString());
+ ret.put("holes", StringUtils.join(holes, ","));
+ return ret;
+ }
+
+ @Override
+ public boolean initFromProperties(Map<String, String> properties) throws LensException {
+ first = TimePartition.of(getUpdatePeriod(), properties.get("first"));
+ latest = TimePartition.of(getUpdatePeriod(), properties.get("latest"));
+ holes = Sets.newTreeSet();
+ String holesStr = properties.get("holes");
+ if (!Strings.isNullOrEmpty(holesStr)) {
+ for (String hole : properties.get("holes").split("\\s*,\\s*")) {
+ holes.add(TimePartition.of(getUpdatePeriod(), hole));
+ }
+ }
+ return isConsistent();
+ }
+
+ private boolean addHole(TimePartition toDrop) {
+ return holes.add(toDrop);
+ }
+
+ private void addHolesBetween(TimePartition begin, TimePartition end, UpdatePeriod updatePeriod) throws LensException {
+ for (Date date : TimeRange.iterable(begin.next().getDate(), end.getDate(), updatePeriod, 1)) {
+ addHole(TimePartition.of(updatePeriod, date));
+ }
+ }
+
+ private TimePartition getNextPartition(TimePartition begin, TimePartition end, int increment) throws LensException {
+ for (Date date : TimeRange.iterable(begin.partitionAtDiff(increment).getDate(),
+ end.partitionAtDiff(increment).getDate(), begin.getUpdatePeriod(), increment)) {
+ TimePartition value = TimePartition.of(begin.getUpdatePeriod(), date);
+ if (!holes.contains(value)) {
+ return value;
+ } else {
+ holes.remove(value);
+ }
+ }
+ return null;
+ }
+
+
+ public boolean isEmpty() {
+ return first == null && latest == null && holes.isEmpty();
+ }
+
+ @Override
+ public boolean isConsistent() {
+ if (first == null && latest != null) {
+ return false;
+ }
+ if (latest == null && first != null) {
+ return false;
+ }
+ for (TimePartition hole : holes) {
+ if (!hole.after(first) || !hole.before(latest)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public boolean exists(TimePartition toCheck) {
+ return !isEmpty() && !toCheck.before(first) && !toCheck.after(latest) && !holes.contains(toCheck);
+ }
+
+ @Override
+ public Iterator<TimePartition> iterator() {
+
+ return new Iterator<TimePartition>() {
+ TimePartition cur = getFirst();
+
+ @Override
+ public boolean hasNext() {
+ return cur != null && getLatest() != null && !cur.after(getLatest());
+ }
+
+ @Override
+ public TimePartition next() {
+ while (holes.contains(cur)) {
+ cur = cur.next();
+ }
+ TimePartition toReturn = cur;
+ cur = cur.next();
+ return toReturn;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
new file mode 100644
index 0000000..64771d0
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
@@ -0,0 +1,207 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata.timeline;
+
+
+import java.util.*;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.cube.metadata.*;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import lombok.Data;
+import lombok.NonNull;
+
+/**
+ * Represents the in-memory data structure that represents timeline of all existing partitions for a given storage
+ * table, update period, partition column. Is an Abstract class. Can be implemented in multiple ways.
+ *
+ * @see org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline
+ * @see org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline
+ */
+@Data
+public abstract class PartitionTimeline implements Iterable<TimePartition> {
+ private final CubeMetastoreClient client;
+ private final String storageTableName;
+ private final UpdatePeriod updatePeriod;
+ private final String partCol;
+ private TreeSet<TimePartition> all;
+
+ /** wrapper on latest data */
+ public Date getLatestDate() {
+ return latest() == null ? null : latest().getDate();
+ }
+
+ /**
+ * Sets PartitionTimeline implementation class's name and specific params in table param.
+ *
+ * @param table
+ * @see #init(org.apache.hadoop.hive.ql.metadata.Table)
+ */
+ public void updateTableParams(Table table) {
+ String prefix = MetastoreUtil.getPartitionInfoKeyPrefix(getUpdatePeriod(), getPartCol());
+ String storageClass = MetastoreUtil.getPartitionTimelineStorageClassKey(getUpdatePeriod(), getPartCol());
+ table.getParameters().put(storageClass, this.getClass().getCanonicalName());
+ for (Map.Entry<String, String> entry : toProperties().entrySet()) {
+ table.getParameters().put(prefix + entry
+ .getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * Extracts timeline implementation class from table params and instantiates it with other arguments, also in table
+ * params.
+ *
+ * @param table
+ * @throws LensException
+ * @see #updateTableParams(org.apache.hadoop.hive.ql.metadata.Table)
+ */
+ public void init(Table table) throws LensException {
+ HashMap<String, String> props = Maps.newHashMap();
+ String prefix = MetastoreUtil.getPartitionInfoKeyPrefix(getUpdatePeriod(), getPartCol());
+ for (Map.Entry<String, String> entry : table.getParameters().entrySet()) {
+ if (entry.getKey().startsWith(prefix)) {
+ props.put(entry.getKey().substring(prefix.length()), entry.getValue());
+ }
+ }
+ initFromProperties(props);
+ }
+
+ /**
+ * Add partition to local memory to be sent for batch addition.
+ *
+ * @see #commitBatchAdditions()
+ */
+ public void addForBatchAddition(TimePartition partition) {
+ if (all == null) {
+ all = Sets.newTreeSet();
+ }
+ all.add(partition);
+ }
+
+ /**
+ * Commit all partitions that were added to batch addition queue. //TODO: improve batch addition implementation.
+ *
+ * @return true if all the partitions were added successfully, or no partitions needed to be added
+ * @throws LensException
+ */
+ public boolean commitBatchAdditions() throws LensException {
+ if (getAll() == null) {
+ return true;
+ }
+ boolean result = add(getAll());
+ all = null;
+ return result;
+ }
+
+ /**
+ * goes to metastore and queries if more partitions exist associated with (partCol = value) in storage table
+ * #getStorageTableName for update period #getUpdatePeriod. This might be useful for implementations while
+ * implementing drop.
+ *
+ * @param value
+ * @return
+ * @throws LensException
+ */
+ public boolean morePartitionsExist(String value) throws LensException {
+ try {
+ return getClient().partitionExistsByFilter(getStorageTableName(), StorageConstants.getPartFilter(getPartCol(),
+ value));
+ } catch (HiveException e) {
+ throw new LensException(e);
+ }
+ }
+
+ /**
+ * Add partition to timeline
+ *
+ * @param partition
+ * @return whether add was successful
+ * @throws LensException
+ */
+ public abstract boolean add(@NonNull TimePartition partition) throws LensException;
+
+ /**
+ * Add multiple partitions to timeline
+ *
+ * @param partitions
+ * @return whether add was successful
+ * @throws LensException
+ */
+ public abstract boolean add(@NonNull Collection<TimePartition> partitions) throws LensException;
+
+ /**
+ * drop partition.
+ *
+ * @param toDrop
+ * @return whether drop was successful
+ * @throws LensException
+ */
+ public abstract boolean drop(@NonNull TimePartition toDrop) throws LensException;
+
+ /**
+ * latest partition. will be null if no partitions exist.
+ *
+ * @return
+ */
+ public abstract TimePartition latest();
+
+ /**
+ * serialize member objects as map
+ *
+ * @return
+ */
+ public abstract Map<String, String> toProperties();
+
+ /**
+ * deserialize member variables from given map
+ *
+ * @param properties
+ * @return true if after deserializing, the timeline is in consistent state
+ * @throws LensException
+ * @see #isConsistent()
+ */
+ public abstract boolean initFromProperties(Map<String, String> properties) throws LensException;
+
+ /**
+ * Whether No partitions have been registered
+ *
+ * @return
+ */
+ public abstract boolean isEmpty();
+
+ /**
+ * whether timeline is in consistent state
+ *
+ * @return
+ */
+ public abstract boolean isConsistent();
+
+ /**
+ * Checks partition existance
+ *
+ * @param partition
+ * @return
+ */
+ public abstract boolean exists(TimePartition partition);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java
new file mode 100644
index 0000000..5626a03
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata.timeline;
+
+
+import java.lang.reflect.Constructor;
+
+import org.apache.lens.cube.metadata.CubeMetastoreClient;
+import org.apache.lens.cube.metadata.MetastoreUtil;
+import org.apache.lens.cube.metadata.UpdatePeriod;
+
+/** factory class for getting new timeline instances */
+public final class PartitionTimelineFactory {
+ private PartitionTimelineFactory() {
+
+ }
+
+ /**
+ * Checks in table params if desired implementing class is given. Otherwise by default returns instance of {@link
+ * org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline}.
+ *
+ * @param client
+ * @param storageTable
+ * @param updatePeriod
+ * @param partitionColumn
+ * @return
+ */
+ public static PartitionTimeline get(CubeMetastoreClient client, String storageTable,
+ UpdatePeriod updatePeriod, String partitionColumn) {
+ try {
+ String storageClassName = client.getTable(storageTable).getParameters().get(
+ MetastoreUtil.getPartitionTimelineStorageClassKey(
+ updatePeriod, partitionColumn));
+ Class<? extends PartitionTimeline> clz = (Class<? extends PartitionTimeline>) Class.forName(storageClassName);
+ Constructor<? extends PartitionTimeline> constructor = clz.getConstructor(
+ CubeMetastoreClient.class, String.class, UpdatePeriod.class, String.class);
+ return constructor.newInstance(
+ client, storageTable, updatePeriod, partitionColumn);
+ } catch (Exception e) {
+ return new EndsAndHolesPartitionTimeline(client, storageTable, updatePeriod, partitionColumn);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
new file mode 100644
index 0000000..8a9450c
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata.timeline;
+
+import java.util.*;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.cube.metadata.CubeMetastoreClient;
+import org.apache.lens.cube.metadata.TimePartition;
+import org.apache.lens.cube.metadata.UpdatePeriod;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import lombok.Data;
+import lombok.NonNull;
+
+/**
+ * Implementation of PartitionTimeline that stores all partitions as a tree set. Created for instrumentation purposes.
+ * Helps in test cases. In real life scenarios, this would be almost always end up causing out of memory error.
+ */
+@Data
+public class StoreAllPartitionTimeline extends PartitionTimeline {
+ TreeSet<TimePartition> allPartitions;
+
+ public StoreAllPartitionTimeline(CubeMetastoreClient client, String storageTableName,
+ UpdatePeriod updatePeriod, String partCol) {
+ super(client, storageTableName, updatePeriod, partCol);
+ allPartitions = Sets.newTreeSet();
+ }
+
+ @Override
+ public boolean add(@NonNull TimePartition partition) throws LensException {
+ return allPartitions.add(partition);
+ }
+
+ @Override
+ public boolean add(@NonNull Collection<TimePartition> partitions) throws LensException {
+ return allPartitions.addAll(partitions);
+ }
+
+ @Override
+ public boolean drop(@NonNull TimePartition toDrop) throws LensException {
+ if (morePartitionsExist(toDrop.getDateString())) {
+ return true;
+ }
+ return allPartitions.remove(toDrop);
+ }
+
+ @Override
+ public TimePartition latest() {
+ return allPartitions.size() == 0 ? null : allPartitions.last();
+ }
+
+ @Override
+ public Map<String, String> toProperties() {
+ HashMap<String, String> map = Maps.newHashMap();
+ map.put("partitions", StringUtils.join(allPartitions, ","));
+ return map;
+ }
+
+ @Override
+ public boolean initFromProperties(Map<String, String> properties) throws LensException {
+ String partitionsStr = properties.get("partitions");
+ if (partitionsStr == null) {
+ return true;
+ }
+ boolean ret = true;
+ for (String s : StringUtils.split(partitionsStr, ",")) {
+ ret &= add(TimePartition.of(getUpdatePeriod(), s));
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return allPartitions.isEmpty();
+ }
+
+ @Override
+ public boolean isConsistent() {
+ return true;
+ }
+
+ @Override
+ public boolean exists(TimePartition partition) {
+ return allPartitions.contains(partition);
+ }
+
+ @Override
+ public Iterator<TimePartition> iterator() {
+ return getAllPartitions().iterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java
index b1a85ad..d5276e0 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java
@@ -21,13 +21,20 @@ package org.apache.lens.cube.parse;
import java.util.*;
+import org.apache.lens.cube.metadata.FactPartition;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
/**
* Collapses the time range filters using IN operators
*/
public class AbridgedTimeRangeWriter implements TimeRangeWriter {
+ //TODO: minimize use of String, use StringBuilders
+
/**
* Return IN clause for the partitions selected in the cube query
*
@@ -44,10 +51,29 @@ public class AbridgedTimeRangeWriter implements TimeRangeWriter {
if (parts == null || parts.isEmpty()) {
return "";
}
-
// Collect partition specs by column in a map
// All filters which contain only a single column will be combined in an IN operator clause
// This clause will be ORed with filters which contain multiple columns.
+ List<String> subFilters = new ArrayList<String>();
+ for (Map.Entry<Set<FactPartition>, Set<FactPartition>> entry : groupPartitions(parts).entrySet()) {
+ List<String> clauses = new ArrayList<String>();
+ String clause;
+ clause = getClause(cubeQueryContext, tableName, entry.getKey());
+ if (clause != null && !clause.isEmpty()) {
+ clauses.add(clause);
+ }
+ clause = getClause(cubeQueryContext, tableName, entry.getValue());
+ if (clause != null && !clause.isEmpty()) {
+ clauses.add(clause);
+ }
+ subFilters.add("(" + StringUtils.join(clauses, " AND ") + ")");
+ }
+ return StringUtils.join(subFilters, " OR ");
+ }
+
+ private String getClause(CubeQueryContext cubeQueryContext,
+ String tableName,
+ Set<FactPartition> parts) throws SemanticException {
Map<String, List<String>> partFilterMap = new HashMap<String, List<String>>();
List<String> allTimeRangeFilters = new ArrayList<String>();
@@ -71,6 +97,53 @@ public class AbridgedTimeRangeWriter implements TimeRangeWriter {
return StringUtils.join(allTimeRangeFilters, " OR ");
}
+ /**
+ * parts is a collection of FactPartition objects. And FactPartition can be viewed as two boolean conditions, one
+ * specified by it's containingPart object, and another specified by itself in the form (partCol = partSpec)
+ * <p/>
+ * Collection of FactPartition objects can be viewed as an OR clause on all the FactPartition objects -- which by
+ * itself is a binary AND clause.
+ * <p/>
+ * So Collection<FactPartition> is nothing but (a AND b) OR (c AND d) OR (e AND f) ...
+ * <p/>
+ * This function tries to reduce such a big clause by using Boolean arithmetic. The big thing it aims to reduce is the
+ * following class of clauses:
+ * <p/>
+ * (a AND c) OR (a AND d) OR (b AND c) OR (b AND d) => ((a OR b) AND (c OR d))
+ * <p/>
+ * Equivalent return value for such a reduction would be an entry in the returned map from set(a,b) to set(c,d).
+ * Assuming the argument was set(a(containing=c), a(containing=d), b(containing=c), b(containing=d))
+ *
+ * @param parts
+ * @return
+ */
+ private Map<Set<FactPartition>, Set<FactPartition>> groupPartitions(Collection<FactPartition> parts) {
+ Map<FactPartition, Set<FactPartition>> partitionSetMap = new HashMap<FactPartition, Set<FactPartition>>();
+ for (FactPartition part : parts) {
+ FactPartition key = part.getContainingPart();
+ part.setContainingPart(null);
+ if (partitionSetMap.get(key) == null) {
+ partitionSetMap.put(key, Sets.<FactPartition>newTreeSet());
+ }
+ partitionSetMap.get(key).add(part);
+ }
+ Map<Set<FactPartition>, Set<FactPartition>> setSetOppositeMap = Maps.newHashMap();
+ for (Map.Entry<FactPartition, Set<FactPartition>> entry : partitionSetMap.entrySet()) {
+ if (setSetOppositeMap.get(entry.getValue()) == null) {
+ setSetOppositeMap.put(entry.getValue(), Sets.<FactPartition>newTreeSet());
+ }
+ if (entry.getKey() != null) {
+ setSetOppositeMap.get(entry.getValue()).add(entry.getKey());
+ }
+ }
+
+ Map<Set<FactPartition>, Set<FactPartition>> setSetMap = Maps.newHashMap();
+ for (Map.Entry<Set<FactPartition>, Set<FactPartition>> entry : setSetOppositeMap.entrySet()) {
+ setSetMap.put(entry.getValue(), entry.getKey());
+ }
+ return setSetMap;
+ }
+
// This takes the output of filter generated by TimeRangeUtils.getTimeRangePartitionFilter
// splits the filters by column names and filters are collected by column name in the
// map passed as argument
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
index 14ea806..6c85c2d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
@@ -22,6 +22,8 @@ import java.util.Iterator;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.lens.cube.metadata.FactPartition;
+
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.parse.SemanticException;