You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/03/10 07:55:53 UTC

[3/3] incubator-lens git commit: LENS-281 : Create Partition Timeline and keep it cached and stored as storage table properties (Rajat Khandelwal via amareshwari)

LENS-281 : Create Partition Timeline and keep it cached and stored as storage table properties (Rajat Khandelwal via amareshwari)


Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/803448a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/803448a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/803448a1

Branch: refs/heads/master
Commit: 803448a18e901a5c3dd9e4a70aa03935be05790b
Parents: 683516c
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Tue Mar 10 12:25:19 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Mar 10 12:25:19 2015 +0530

----------------------------------------------------------------------
 .../metadata/CaseInsensitiveStringHashMap.java  |  55 +++
 .../lens/cube/metadata/CubeFactTable.java       |   6 +-
 .../lens/cube/metadata/CubeMetastoreClient.java | 472 +++++++++++++++---
 .../lens/cube/metadata/FactPartition.java       | 163 +++++++
 .../lens/cube/metadata/MetastoreConstants.java  |   2 +
 .../lens/cube/metadata/MetastoreUtil.java       |  22 +-
 .../org/apache/lens/cube/metadata/Storage.java  |  61 +--
 .../lens/cube/metadata/StorageConstants.java    |   4 +
 .../lens/cube/metadata/TimePartition.java       | 126 +++++
 .../apache/lens/cube/metadata/UpdatePeriod.java |   2 +-
 .../timeline/EndsAndHolesPartitionTimeline.java | 213 +++++++++
 .../metadata/timeline/PartitionTimeline.java    | 207 ++++++++
 .../timeline/PartitionTimelineFactory.java      |  59 +++
 .../timeline/StoreAllPartitionTimeline.java     | 111 +++++
 .../cube/parse/AbridgedTimeRangeWriter.java     |  75 ++-
 .../lens/cube/parse/BetweenTimeRangeWriter.java |   2 +
 .../apache/lens/cube/parse/FactPartition.java   | 166 -------
 .../lens/cube/parse/ORTimeRangeWriter.java      |   2 +
 .../lens/cube/parse/StorageTableResolver.java   | 263 +++++-----
 .../org/apache/lens/cube/parse/StorageUtil.java |  15 +
 .../org/apache/lens/cube/parse/TimeRange.java   |  87 ++++
 .../apache/lens/cube/parse/TimeRangeUtils.java  |   2 +
 .../apache/lens/cube/parse/TimeRangeWriter.java |   2 +
 .../cube/metadata/TestCubeMetastoreClient.java  | 476 +++++++++++--------
 .../lens/cube/metadata/TestTimePartition.java   |  76 +++
 .../apache/lens/cube/parse/CubeTestSetup.java   | 121 ++++-
 .../lens/cube/parse/TestCubeRewriter.java       |  13 +-
 .../apache/lens/cube/parse/TestStorageUtil.java |   1 +
 .../lens/cube/parse/TestTimeRangeWriter.java    |   1 +
 .../metastore/CubeMetastoreServiceImpl.java     |  37 +-
 .../server/metastore/TestMetastoreService.java  |   6 +-
 31 files changed, 2164 insertions(+), 684 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CaseInsensitiveStringHashMap.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CaseInsensitiveStringHashMap.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CaseInsensitiveStringHashMap.java
new file mode 100644
index 0000000..e125956
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CaseInsensitiveStringHashMap.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * HashMap from String to type T where the key is case insensitive.
+ *
+ * @param <T>
+ */
+public class CaseInsensitiveStringHashMap<T> extends HashMap<String, T> {
+  @Override
+  public T get(Object key) {
+    return super.get(caseInsensitiveKey(key));
+  }
+
+  @Override
+  public T put(String key, T value) {
+    return super.put(caseInsensitiveKey(key), value);
+  }
+
+  @Override
+  public boolean containsKey(Object key) {
+    return super.containsKey(caseInsensitiveKey(key));
+  }
+
+  private static String caseInsensitiveKey(Object key) {
+    return key == null ? null : key.toString().toLowerCase();
+  }
+
+  @Override
+  public void putAll(Map<? extends String, ? extends T> m) {
+    for (Map.Entry<? extends String, ? extends T> entry : m.entrySet()) {
+      put(entry.getKey(), entry.getValue());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
index b3ed6ef..e0e87ab 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
@@ -179,7 +179,8 @@ public final class CubeFactTable extends AbstractCubeTable {
     // between the dates
     UpdatePeriodComparator cmp = new UpdatePeriodComparator();
     for (UpdatePeriod i : updatePeriods) {
-      if (UpdatePeriod.YEARLY == i || UpdatePeriod.QUARTERLY == i || UpdatePeriod.MONTHLY == i) {
+      if (UpdatePeriod.YEARLY == i || UpdatePeriod.QUARTERLY == i || UpdatePeriod.MONTHLY == i
+        || UpdatePeriod.WEEKLY == i) {
         int intervals = 0;
         switch (i) {
         case YEARLY:
@@ -226,8 +227,7 @@ public final class CubeFactTable extends AbstractCubeTable {
   }
 
   /**
-   * Return valid columns of the fact, which can be specified by the property
-   * MetastoreUtil.getValidColumnsKey(getName())
+   * Return valid columns of the fact, which can be specified by property MetastoreUtil.getValidColumnsKey(getName())
    *
    * @return
    */

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 7e6fe7e..2e03df1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -22,16 +22,17 @@ package org.apache.lens.cube.metadata;
 import java.text.ParseException;
 import java.util.*;
 
+import org.apache.lens.api.LensException;
 import org.apache.lens.cube.metadata.Storage.LatestInfo;
 import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo;
+import org.apache.lens.cube.metadata.timeline.PartitionTimeline;
+import org.apache.lens.cube.metadata.timeline.PartitionTimelineFactory;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.Partition;
@@ -39,9 +40,13 @@ import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.thrift.TException;
 
+import com.google.common.collect.Maps;
+import lombok.extern.apachecommons.CommonsLog;
+
 /**
  * Wrapper class around Hive metastore to do cube metastore operations.
  */
+@CommonsLog
 public class CubeMetastoreClient {
   private final HiveConf config;
   private final boolean enableCaching;
@@ -52,24 +57,268 @@ public class CubeMetastoreClient {
   }
 
   // map from table name to Table
-  private final Map<String, Table> allHiveTables = new HashMap<String, Table>();
+  private final Map<String, Table> allHiveTables = Maps.newHashMap();
   // map from dimension name to Dimension
-  private final Map<String, Dimension> allDims = new HashMap<String, Dimension>();
+  private final Map<String, Dimension> allDims = Maps.newHashMap();
   // map from cube name to Cube
-  private final Map<String, CubeInterface> allCubes = new HashMap<String, CubeInterface>();
+  private final Map<String, CubeInterface> allCubes = Maps.newHashMap();
   // map from dimtable name to CubeDimensionTable
-  private final Map<String, CubeDimensionTable> allDimTables = new HashMap<String, CubeDimensionTable>();
+  private final Map<String, CubeDimensionTable> allDimTables = Maps.newHashMap();
   // map from fact name to fact table
-  private final Map<String, CubeFactTable> allFactTables = new HashMap<String, CubeFactTable>();
+  private final Map<String, CubeFactTable> allFactTables = Maps.newHashMap();
   // map from storage name to storage
-  private final Map<String, Storage> allStorages = new HashMap<String, Storage>();
-
+  private final Map<String, Storage> allStorages = Maps.newHashMap();
+  // Partition cache. Inner class since it logically belongs here
+  PartitionTimelineCache partitionTimelineCache = new PartitionTimelineCache();
   // dbname to client mapping
-  private static final Map<String, CubeMetastoreClient> CLIENT_MAPPING =
-    new HashMap<String, CubeMetastoreClient>();
-
+  private static final Map<String, CubeMetastoreClient> CLIENT_MAPPING = Maps.newHashMap();
   private SchemaGraph schemaGraph;
 
+  /** extract storage name from fact and storage table name. String operation */
+  private String extractStorageName(CubeFactTable fact, String storageTableName) throws LensException {
+    int ind = storageTableName.lastIndexOf(fact.getName());
+    if (ind <= 0) {
+      throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName());
+    }
+    return storageTableName.substring(0, ind - StorageConstants.STORGAE_SEPARATOR.length());
+  }
+
+  /**
+   * get latest date for timeDimension from all fact-storage tables belonging to the given cube having timeDimension,
+   * return the most recent of all.
+   * <p/>
+   * latest date for a single fact-storage table for given time dimension is the latest of the latest dates for all its
+   * update periods
+   *
+   * @param cube
+   * @param timeDimension
+   * @return
+   * @throws HiveException
+   * @throws LensException
+   */
+  public Date getLatestDateOfCube(Cube cube, String timeDimension) throws HiveException, LensException {
+    String partCol = cube.getPartitionColumnOfTimeDim(timeDimension);
+    Date max = new Date(Long.MIN_VALUE);
+    boolean updated = false;
+    for (CubeFactTable fact : getAllFactTables(cube)) {
+      for (String storage : fact.getStorages()) {
+        for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) {
+          PartitionTimeline timeline = partitionTimelineCache.get(fact.getName(), storage, updatePeriod,
+            partCol);
+          if (timeline != null) {// this storage table is partitioned by partCol or not.
+            Date latest = timeline.getLatestDate();
+            if (latest != null && latest.after(max)) {
+              max = latest;
+              updated = true;
+            }
+          }
+        }
+      }
+    }
+    return updated ? max : null;
+  }
+
+  /** clear hive table cache */
+  public void clearHiveTableCache() {
+    allHiveTables.clear();
+  }
+
+  /**
+   * In-memory storage of {@link org.apache.lens.cube.metadata.timeline.PartitionTimeline} objects for each valid
+   * storagetable-updateperiod-partitioncolumn tuple. also simultaneously stored in metastore table of the
+   * storagetable.
+   */
+  class PartitionTimelineCache extends CaseInsensitiveStringHashMap<// storage table
+    TreeMap<UpdatePeriod,
+      CaseInsensitiveStringHashMap<// partition column
+        PartitionTimeline>>> {
+    /**
+     * Returns true if all the timelines for fact-storage table are empty for all valid update periods.
+     *
+     * @param fact
+     * @param storage
+     * @param partCol
+     * @return
+     * @throws HiveException
+     * @throws LensException
+     */
+    public boolean noPartitionsExist(String fact, String storage, String partCol)
+      throws HiveException, LensException {
+      if (get(fact, storage) == null) {
+        return true;
+      }
+      for (UpdatePeriod updatePeriod : get(fact, storage).keySet()) {
+        PartitionTimeline timeline = get(fact, storage, updatePeriod, partCol);
+        if (timeline != null && !timeline.isEmpty()) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+    /**
+     * get all timelines for all update periods and partition columns for the given fact-storage pair. If already loaded
+     * in memory, it'll return that. If not, it'll first try to load it from table properties. If not found in table
+     * properties, it'll get all partitions, compute timelines in memory, write back all loads timelines to table
+     * properties for further usage and return them.
+     *
+     * @param fact
+     * @param storage
+     * @return
+     * @throws HiveException
+     * @throws LensException
+     */
+    public TreeMap<UpdatePeriod, CaseInsensitiveStringHashMap<PartitionTimeline>> get(String fact, String storage)
+      throws HiveException, LensException {
+      String storageTableName = MetastoreUtil.getStorageTableName(fact, Storage.getPrefix(storage));
+      if (get(storageTableName) == null) {
+        // not found in memory, try loading from table properties.
+        Table storageTable = getTable(storageTableName);
+        if (storageTable.getParameters().get(MetastoreUtil.getPartitoinTimelineCachePresenceKey()) == null) {
+          // Not found in table properties either, compute from all partitions of the fact-storage table.
+          // First make sure all combinations of update period and partition column have an entry even
+          // if no partitions exist
+          if (getCubeFact(fact).getUpdatePeriods() != null && getCubeFact(fact).getUpdatePeriods().get(
+            storage) != null) {
+            for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
+              for (String partCol : getTimePartsOfTable(storageTable)) {
+                partitionTimelineCache.ensureEntry(storageTableName, updatePeriod, partCol);
+              }
+            }
+          }
+          // Then add all existing partitions for batch addition in respective timelines.
+          List<String> timeParts = getTimePartsOfTable(storageTable);
+          List<FieldSchema> partCols = storageTable.getPartCols();
+          for (Partition partition : getPartitionsByFilter(storageTableName, null)) {
+            UpdatePeriod period = deduceUpdatePeriod(partition);
+            List<String> values = partition.getValues();
+            for (int i = 0; i < partCols.size(); i++) {
+              if (timeParts.contains(partCols.get(i).getName())) {
+                partitionTimelineCache.addForBatchAddition(storageTableName, period, partCols.get(i).getName(),
+                  values.get(i));
+              }
+            }
+          }
+          // commit all batch addition for the storage table, which will in-turn commit all batch additions in all it's
+          // timelines.
+          commitAllBatchAdditions(storageTableName);
+        } else {
+          // found in table properties, load from there.
+          for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
+            for (String partCol : getTimePartsOfTable(storageTableName)) {
+              ensureEntry(storageTableName, updatePeriod, partCol).init(storageTable);
+            }
+          }
+        }
+      }
+      // return the final value from memory
+      return get(storageTableName);
+    }
+
+    /**
+     * Adds given partition(for storageTable, updatePeriod, partitionColum=partition) for batch addition in an
+     * appropriate timeline object. Ignore if partition is not valid.
+     *
+     * @param storageTable
+     * @param updatePeriod
+     * @param partitionColumn
+     * @param partition
+     */
+    public void addForBatchAddition(String storageTable, UpdatePeriod updatePeriod, String partitionColumn,
+      String partition) {
+      try {
+        ensureEntry(storageTable, updatePeriod, partitionColumn).addForBatchAddition(TimePartition.of(updatePeriod,
+          partition));
+      } catch (LensException e) {
+        // to take care of the case where partition name is something like `latest`
+        log.error("Couldn't parse partition: " + partition + " with update period: " + updatePeriod + ", skipping.", e);
+      }
+    }
+
+    /**
+     * helper method for ensuring get(storageTable).get(updatePeriod).get(partitionColumn) gives a non-null object.
+     * <p/>
+     * kind of like mkdir -p
+     *
+     * @param storageTable
+     * @param updatePeriod
+     * @param partitionColumn
+     * @return timeline if already exists, or puts a new timeline and returns.
+     */
+    public PartitionTimeline ensureEntry(String storageTable, UpdatePeriod updatePeriod, String partitionColumn) {
+      if (get(storageTable) == null) {
+        put(storageTable, new TreeMap<UpdatePeriod, CaseInsensitiveStringHashMap<PartitionTimeline>>());
+      }
+      if (get(storageTable).get(updatePeriod) == null) {
+        get(storageTable).put(updatePeriod, new CaseInsensitiveStringHashMap<PartitionTimeline>());
+      }
+      if (get(storageTable).get(updatePeriod).get(partitionColumn) == null) {
+        get(storageTable).get(updatePeriod).put(partitionColumn, PartitionTimelineFactory.get(
+          CubeMetastoreClient.this, storageTable, updatePeriod, partitionColumn));
+      }
+      return get(storageTable).get(updatePeriod).get(partitionColumn);
+    }
+
+    /**
+     * commit all batch addition for all its timelines.
+     *
+     * @param storageTable
+     * @throws HiveException
+     * @throws LensException
+     */
+    public void commitAllBatchAdditions(String storageTable) throws HiveException, LensException {
+      if (get(storageTable) != null) {
+        for (UpdatePeriod updatePeriod : get(storageTable).keySet()) {
+          for (String partCol : get(storageTable).get(updatePeriod).keySet()) {
+            PartitionTimeline timeline = get(storageTable).get(updatePeriod).get(partCol);
+            timeline.commitBatchAdditions();
+          }
+        }
+        alterTablePartitionCache(storageTable);
+      }
+    }
+
+    /** check partition existence in the appropriate timeline if it exists */
+    public boolean partitionTimeExists(String name, String storage, UpdatePeriod period, String partCol, Date partSpec)
+      throws HiveException, LensException {
+      return get(name, storage, period, partCol) != null && get(name, storage, period, partCol).exists(TimePartition.of(
+        period, partSpec));
+    }
+
+    /**
+     * returns the timeline corresponding to fact-storage table, updatePeriod, partCol. null if doesn't exist, which
+     * would only happen if the combination is not valid/supported
+     */
+    public PartitionTimeline get(String fact, String storage, UpdatePeriod updatePeriod, String partCol)
+      throws HiveException, LensException {
+      return get(fact, storage) != null && get(fact, storage).get(updatePeriod) != null && get(fact, storage).get(
+        updatePeriod).get(
+        partCol) != null ? get(fact, storage).get(updatePeriod).get(partCol) : null;
+    }
+
+    /** update partition timeline cache for addition of time partition */
+    public void updateForAddition(String cubeTableName, String storageName, UpdatePeriod updatePeriod,
+      Map<String, Date> timePartSpec)
+      throws HiveException, LensException {
+      for (Map.Entry<String, Date> entry : timePartSpec.entrySet()) {
+        //Assume timelines has all the time part columns.
+        get(cubeTableName, storageName, updatePeriod, entry.getKey()).add(TimePartition.of(updatePeriod,
+          entry.getValue()));
+      }
+    }
+
+    /** update partition timeline cache for deletion of time partition */
+    public void updateForDeletion(String cubeTableName, String storageName, UpdatePeriod updatePeriod,
+      Map<String, Date> timePartSpec) throws HiveException, LensException {
+      for (Map.Entry<String, Date> entry : timePartSpec.entrySet()) {
+        get(cubeTableName, storageName, updatePeriod, entry.getKey()).drop(TimePartition.of(
+          updatePeriod, entry.getValue()));
+      }
+    }
+
+  }
+
+
   /**
    * Get the instance of {@link CubeMetastoreClient} corresponding to {@link HiveConf}
    *
@@ -359,20 +608,62 @@ public class CubeMetastoreClient {
 
   /**
    * Add a partition specified by the storage partition desc on the storage passed.
+   * <p/>
+   * TODO: separate methods for fact and dim partition addition.
    *
    * @param partSpec    The storage partition description
    * @param storageName The storage object
    * @throws HiveException
    */
-  public void addPartition(StoragePartitionDesc partSpec, String storageName) throws HiveException {
-    String storageTableName =
-      MetastoreUtil.getStorageTableName(partSpec.getCubeTableName(), Storage.getPrefix(storageName));
+  public void addPartition(StoragePartitionDesc partSpec, String storageName) throws HiveException, LensException {
+    String storageTableName = MetastoreUtil.getStorageTableName(partSpec.getCubeTableName(), Storage.getPrefix(
+      storageName));
+    if (getDimensionTable(partSpec.getCubeTableName()) != null) {
+      // Adding partition in dimension table.
+      getStorage(storageName).addPartition(getClient(), partSpec,
+        getDimTableLatestInfo(storageTableName, partSpec.getTimePartSpec(), partSpec.getUpdatePeriod())
+      );
+    } else {
+      // first update in memory, then add to hive table's partitions. delete is reverse.
+      partitionTimelineCache.updateForAddition(partSpec.getCubeTableName(), storageName, partSpec.getUpdatePeriod(),
+        partSpec.getTimePartSpec());
+      // Adding partition in fact table.
+      getStorage(storageName).addPartition(getClient(), partSpec, null);
+      // update hive table
+      alterTablePartitionCache(MetastoreUtil.getStorageTableName(partSpec.getCubeTableName(), Storage.getPrefix(
+        storageName)));
+    }
+  }
 
-    getStorage(storageName).addPartition(getClient(), partSpec,
-      getLatestInfo(storageTableName, partSpec.getTimePartSpec(), partSpec.getUpdatePeriod()));
+  /**
+   * store back all timelines of given storage table to table properties
+   *
+   * @param storageTableName
+   * @throws HiveException
+   */
+  private void alterTablePartitionCache(String storageTableName) throws HiveException {
+    Table table = getTable(storageTableName);
+    Map<String, String> params = table.getParameters();
+    if (partitionTimelineCache.get(storageTableName) != null) {
+      for (UpdatePeriod updatePeriod : partitionTimelineCache.get(storageTableName).keySet()) {
+        for (Map.Entry<String, PartitionTimeline> entry : partitionTimelineCache.get(storageTableName)
+          .get(updatePeriod).entrySet()) {
+          entry.getValue().updateTableParams(table);
+        }
+      }
+      params.put(MetastoreUtil.getPartitoinTimelineCachePresenceKey(), "true");
+      alterHiveTable(storageTableName, table);
+    }
+  }
+
+  /** extract update period from partition properties */
+  private UpdatePeriod deduceUpdatePeriod(Partition partition) {
+    return UpdatePeriod.valueOf(partition.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD));
   }
 
-  public void addPartitions(List<StoragePartitionDesc> storagePartitionDescs, String storageName) throws HiveException {
+  /** batch addition */
+  public void addPartitions(List<StoragePartitionDesc> storagePartitionDescs, String storageName)
+    throws HiveException, LensException {
     //TODO: improve this in later jira. Just providing naive implementation for now.
     // Should ideally do some optimization because the list has been provided together, not one by one.
     for (StoragePartitionDesc partSpec : storagePartitionDescs) {
@@ -380,7 +671,7 @@ public class CubeMetastoreClient {
     }
   }
 
-  private LatestInfo getLatestInfo(String storageTableName, Map<String, Date> partitionTimestamps,
+  private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, Date> partitionTimestamps,
     UpdatePeriod updatePeriod) throws HiveException {
     Table hiveTable = getHiveTable(storageTableName);
     String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS);
@@ -394,7 +685,7 @@ public class CubeMetastoreClient {
         }
         boolean makeLatest = true;
         Partition part = getLatestPart(storageTableName, partCol);
-        Date latestTimestamp = getLatestTimeStamp(part, partCol);
+        Date latestTimestamp = getLatestTimeStampOfDimtable(part, partCol);
         if (latestTimestamp != null && pTimestamp.before(latestTimestamp)) {
           makeLatest = false;
         }
@@ -411,11 +702,11 @@ public class CubeMetastoreClient {
     }
   }
 
-  private boolean isLatestPart(Partition part) {
+  private boolean isLatestPartOfDimtable(Partition part) {
     return part.getValues().contains(StorageConstants.LATEST_PARTITION_VALUE);
   }
 
-  public Date getLatestTimeStamp(Partition part, String partCol) throws HiveException {
+  public Date getLatestTimeStampOfDimtable(Partition part, String partCol) throws HiveException {
     if (part != null) {
       String latestTimeStampStr = part.getParameters().get(MetastoreUtil.getLatestPartTimestampKey(partCol));
       String latestPartUpdatePeriod = part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD);
@@ -444,7 +735,8 @@ public class CubeMetastoreClient {
     return partDate;
   }
 
-  private LatestInfo getNextLatest(Table hiveTable, String timeCol, final int timeColIndex) throws HiveException {
+  private LatestInfo getNextLatestOfDimtable(Table hiveTable, String timeCol, final int timeColIndex)
+    throws HiveException {
     // getClient().getPartitionsByNames(tbl, partNames)
     List<Partition> partitions = getClient().getPartitions(hiveTable);
 
@@ -469,7 +761,7 @@ public class CubeMetastoreClient {
       }
     });
     for (Partition part : partitions) {
-      if (!isLatestPart(part)) {
+      if (!isLatestPartOfDimtable(part)) {
         Date partDate = getPartDate(part, timeColIndex);
         if (partDate != null) {
           allPartTimeVals.add(part);
@@ -502,7 +794,7 @@ public class CubeMetastoreClient {
    * @throws HiveException
    */
   public void dropPartition(String cubeTableName, String storageName, Map<String, Date> timePartSpec,
-    Map<String, String> nonTimePartSpec, UpdatePeriod updatePeriod) throws HiveException {
+    Map<String, String> nonTimePartSpec, UpdatePeriod updatePeriod) throws HiveException, LensException {
     String storageTableName = MetastoreUtil.getStorageTableName(cubeTableName, Storage.getPrefix(storageName));
     Table hiveTable = getHiveTable(storageTableName);
     List<FieldSchema> partCols = hiveTable.getPartCols();
@@ -518,47 +810,55 @@ public class CubeMetastoreClient {
         throw new HiveException("Invalid partspec, missing value for" + column.getName());
       }
     }
-    String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS);
-    Map<String, LatestInfo> latest = new HashMap<String, Storage.LatestInfo>();
-    if (timePartColsStr != null) {
-      List<String> timePartCols = Arrays.asList(StringUtils.split(timePartColsStr, ','));
-      for (String timeCol : timePartSpec.keySet()) {
-        if (!timePartCols.contains(timeCol)) {
-          throw new HiveException("Not a time partition column:" + timeCol);
-        }
-        int timeColIndex = partColNames.indexOf(timeCol);
-        Partition part = getLatestPart(storageTableName, timeCol);
-
-        // check if partition being dropped is the latest partition
-        boolean isLatest = true;
-        for (int i = 0; i < partVals.size(); i++) {
-          if (i != timeColIndex) {
-            if (!part.getValues().get(i).equals(partVals.get(i))) {
-              isLatest = false;
-              break;
-            }
+    if (isDimensionTable(cubeTableName)) {
+      String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS);
+      Map<String, LatestInfo> latest = new HashMap<String, Storage.LatestInfo>();
+      if (timePartColsStr != null) {
+        List<String> timePartCols = Arrays.asList(StringUtils.split(timePartColsStr, ','));
+        for (String timeCol : timePartSpec.keySet()) {
+          if (!timePartCols.contains(timeCol)) {
+            throw new HiveException("Not a time partition column:" + timeCol);
           }
-        }
-        if (isLatest) {
-          Date latestTimestamp = getLatestTimeStamp(part, timeCol);
-          Date dropTimestamp;
-          try {
-            dropTimestamp = updatePeriod.format().parse(updatePeriod.format().format(timePartSpec.get(timeCol)));
-          } catch (ParseException e) {
-            throw new HiveException(e);
+          int timeColIndex = partColNames.indexOf(timeCol);
+          Partition part = getLatestPart(storageTableName, timeCol);
+
+          // check if partition being dropped is the latest partition
+          boolean isLatest = true;
+          for (int i = 0; i < partVals.size(); i++) {
+            if (i != timeColIndex) {
+              if (!part.getValues().get(i).equals(partVals.get(i))) {
+                isLatest = false;
+                break;
+              }
+            }
           }
-          if (latestTimestamp != null && dropTimestamp.equals(latestTimestamp)) {
-            LatestInfo latestInfo = getNextLatest(hiveTable, timeCol, timeColIndex);
-            latest.put(timeCol, latestInfo);
+          if (isLatest) {
+            Date latestTimestamp = getLatestTimeStampOfDimtable(part, timeCol);
+            Date dropTimestamp;
+            try {
+              dropTimestamp = updatePeriod.format().parse(updatePeriod.format().format(timePartSpec.get(timeCol)));
+            } catch (ParseException e) {
+              throw new HiveException(e);
+            }
+            if (latestTimestamp != null && dropTimestamp.equals(latestTimestamp)) {
+              LatestInfo latestInfo = getNextLatestOfDimtable(hiveTable, timeCol, timeColIndex);
+              latest.put(timeCol, latestInfo);
+            }
           }
         }
+      } else {
+        if (timePartSpec != null && !timePartSpec.isEmpty()) {
+          throw new HiveException("Not time part columns" + timePartSpec.keySet());
+        }
       }
+      getStorage(storageName).dropPartition(getClient(), storageTableName, partVals,
+        latest);
     } else {
-      if (timePartSpec != null && !timePartSpec.isEmpty()) {
-        throw new HiveException("Not time part columns" + timePartSpec.keySet());
-      }
+      // dropping fact partition
+      getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, null);
+      partitionTimelineCache.updateForDeletion(cubeTableName, storageName, updatePeriod, timePartSpec);
+      this.alterTablePartitionCache(storageTableName);
     }
-    getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, latest);
   }
 
   private Map<String, String> getPartitionSpec(UpdatePeriod updatePeriod, Map<String, Date> partitionTimestamps) {
@@ -578,7 +878,15 @@ public class CubeMetastoreClient {
     }
   }
 
-  boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod,
+  /** extract storage name and check in timeline cache for existance */
+  public boolean factPartitionExists(CubeFactTable fact, FactPartition part, String storageTableName)
+    throws HiveException, LensException {
+    String storage = extractStorageName(fact, storageTableName);
+    return partitionTimelineCache.partitionTimeExists(fact.getName(), storage, part.getPeriod(), part.getPartCol(),
+      part.getPartSpec());
+  }
+
+  public boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod,
     Map<String, Date> partitionTimestamp, Map<String, String> partSpec) throws HiveException {
     String storageTableName = MetastoreUtil.getFactStorageTableName(factName, storageName);
     return partitionExists(storageTableName, updatePeriod, partitionTimestamp, partSpec);
@@ -611,8 +919,7 @@ public class CubeMetastoreClient {
     }
   }
 
-  public int getNumPartitionsByFilter(String storageTableName, String filter) throws MetaException,
-    NoSuchObjectException, HiveException, TException {
+  public int getNumPartitionsByFilter(String storageTableName, String filter) throws HiveException, TException {
     return getClient().getNumPartitionsByFilter(getTable(storageTableName), filter);
   }
 
@@ -639,9 +946,14 @@ public class CubeMetastoreClient {
       partitionTimestamps);
   }
 
-  boolean latestPartitionExists(String factName, String storageName, String latestPartCol) throws HiveException {
+  boolean latestPartitionExists(String factName, String storageName, String latestPartCol)
+    throws HiveException, LensException {
     String storageTableName = MetastoreUtil.getFactStorageTableName(factName, storageName);
-    return partitionExistsByFilter(storageTableName, StorageConstants.getLatestPartFilter(latestPartCol));
+    if (isDimensionTable(factName)) {
+      return partitionExistsByFilter(storageTableName, StorageConstants.getLatestPartFilter(latestPartCol));
+    } else {
+      return !partitionTimelineCache.noPartitionsExist(factName, storageName, latestPartCol);
+    }
   }
 
   Partition getLatestPart(String storageTableName, String latestPartCol) throws HiveException {
@@ -664,7 +976,18 @@ public class CubeMetastoreClient {
     return getTable(tableName);
   }
 
-  private Table getTable(String tableName) throws HiveException {
+  private List<String> getTimePartsOfTable(String storageTableName) throws HiveException {
+    return getTimePartsOfTable(getTable(storageTableName));
+  }
+
+  /** extract from table properties */
+  public List<String> getTimePartsOfTable(Table table) {
+    List<String> ret = Arrays.asList(StringUtils.split(table.getParameters().get(MetastoreConstants.TIME_PART_COLUMNS),
+      ","));
+    return ret == null ? new ArrayList<String>() : ret;
+  }
+
+  public Table getTable(String tableName) throws HiveException {
     Table tbl;
     try {
       tbl = allHiveTables.get(tableName.toLowerCase());
@@ -818,8 +1141,7 @@ public class CubeMetastoreClient {
    */
 
   public CubeFactTable getFactTable(String tableName) throws HiveException {
-    Table tbl = getTable(tableName);
-    return getFactTable(tbl);
+    return getFactTable(getTable(tableName));
   }
 
   private CubeFactTable getFactTable(Table tbl) throws HiveException {
@@ -928,8 +1250,7 @@ public class CubeMetastoreClient {
   public CubeFactTable getCubeFact(String tableName) throws HiveException {
     CubeFactTable fact = allFactTables.get(tableName.toLowerCase());
     if (fact == null) {
-      Table tbl = getTable(tableName);
-      fact = getFactTable(tbl);
+      fact = getFactTable(tableName);
       if (enableCaching) {
         allFactTables.put(tableName.toLowerCase(), fact);
       }
@@ -1198,8 +1519,11 @@ public class CubeMetastoreClient {
     return columnsChanged;
   }
 
-  private void alterHiveTable(String table, Table hiveTable, List<FieldSchema> columns) throws HiveException {
-    hiveTable.getTTable().getSd().setCols(columns);
+  public void pushHiveTable(Table hiveTable) throws HiveException {
+    alterHiveTable(hiveTable.getTableName(), hiveTable);
+  }
+
+  public void alterHiveTable(String table, Table hiveTable) throws HiveException {
     try {
       getClient().alterTable(table, hiveTable);
     } catch (InvalidOperationException e) {
@@ -1207,10 +1531,16 @@ public class CubeMetastoreClient {
     }
     if (enableCaching) {
       // refresh the table in cache
+      //TODO: don't need to fetch again. can put table -> hiveTable
       refreshTable(table);
     }
   }
 
+  private void alterHiveTable(String table, Table hiveTable, List<FieldSchema> columns) throws HiveException {
+    hiveTable.getTTable().getSd().setCols(columns);
+    alterHiveTable(table, hiveTable);
+  }
+
   /**
    * Alter cube specified by the name to new definition
    *

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
new file mode 100644
index 0000000..729da30
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.cube.metadata;
+
+import java.text.DateFormat;
+import java.util.Date;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
+
+@EqualsAndHashCode
+public class FactPartition implements Comparable<FactPartition> {
+  @Getter
+  private final String partCol;
+  @Getter
+  private final Date partSpec;
+  @Getter
+  private final Set<String> storageTables = new LinkedHashSet<String>();
+  @Getter
+  private final UpdatePeriod period;
+  @Getter
+  @Setter
+  private FactPartition containingPart;
+  private final DateFormat partFormat;
+
+  public FactPartition(String partCol, Date partSpec, UpdatePeriod period,
+    FactPartition containingPart, DateFormat partFormat) {
+    this.partCol = partCol;
+    this.partSpec = partSpec;
+    this.period = period;
+    this.containingPart = containingPart;
+    this.partFormat = partFormat;
+  }
+
+  public FactPartition(String partCol, Date partSpec, UpdatePeriod period, FactPartition containingPart,
+    DateFormat partFormat, Set<String> storageTables) {
+    this(partCol, partSpec, period, containingPart, partFormat);
+    this.storageTables.addAll(storageTables);
+  }
+
+  public boolean hasContainingPart() {
+    return containingPart != null;
+  }
+
+  public boolean found() {
+    return !getStorageTables().isEmpty();
+  }
+
+  public String getFormattedPartSpec() {
+    if (partFormat == null) {
+      return getPartString();
+    } else {
+      return partFormat.format(partSpec);
+    }
+  }
+
+  public String getPartString() {
+    return period.format().format(partSpec);
+  }
+
+  public String getFormattedFilter(String tableName) {
+    return getFormattedFilter(partCol, tableName);
+  }
+
+  public String getFormattedFilter(String partCol, String tableName) {
+    StringBuilder builder = new StringBuilder();
+    if (containingPart != null) {
+      builder.append(containingPart.getFormattedFilter(tableName));
+      builder.append(" AND ");
+    }
+    if (tableName != null) {
+      builder.append(tableName);
+      builder.append(".");
+    }
+    builder.append(partCol);
+    builder.append(" = '").append(getFormattedPartSpec()).append("'");
+    return builder.toString();
+  }
+
+  public String getFilter() {
+    StringBuilder builder = new StringBuilder();
+    if (containingPart != null) {
+      builder.append(containingPart.getFilter());
+      builder.append(" AND ");
+    }
+    builder.append(partCol);
+    builder.append(" = '").append(getPartString()).append("'");
+    return builder.toString();
+  }
+
+  @Override
+  public String toString() {
+    return getFilter();
+  }
+
+  public int compareTo(FactPartition o) {
+    int colComp = this.partCol.compareTo(o.partCol);
+    if (colComp == 0) {
+      int partComp = 0;
+      if (this.partSpec != null) {
+        if (o.partSpec == null) {
+          partComp = 1;
+        }
+        partComp = this.partSpec.compareTo(o.partSpec);
+      } else {
+        if (o.partSpec != null) {
+          partComp = -1;
+        } else {
+          partComp = 0;
+        }
+      }
+      if (partComp == 0) {
+        int upComp = 0;
+        if (this.period != null && o.period != null) {
+          upComp = this.period.compareTo(o.period);
+        } else if (this.period == null && o.period == null) {
+          upComp = 0;
+        } else if (this.period == null) {
+          upComp = -1;
+        } else {
+          upComp = 1;
+        }
+        if (upComp == 0) {
+          if (this.containingPart != null) {
+            if (o.containingPart == null) {
+              return 1;
+            }
+            return this.containingPart.compareTo(o.containingPart);
+          } else {
+            if (o.containingPart != null) {
+              return -1;
+            } else {
+              return 0;
+            }
+          }
+        }
+        return upComp;
+      }
+      return partComp;
+    }
+    return colComp;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
index 979de6d..7e6360e 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
@@ -97,6 +97,8 @@ public final class MetastoreConstants {
   // storage constants
   public static final String STORAGE_ENTITY_PFX = "storage.";
   public static final String STORAGE_PFX = "cube.storagetable.";
+  public static final String PARTITION_TIMELINE_CACHE = "partition.timeline.cache.";
+  public static final String STORAGE_CLASS = "storage.class";
   public static final String TIME_PART_COLUMNS = "cube.storagetable.time.partcols";
   public static final String LATEST_PART_TIMESTAMP_SFX = ".latest.part.timestamp";
   public static final String PARTITION_UPDATE_PERIOD_SFX = ".partition.update.period";

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index c21c72a..0be8e5f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -347,16 +347,16 @@ public class MetastoreUtil {
   // Utils ///
   // /////////////////////////
   public static <E extends Named> String getNamedStr(Collection<E> set) {
-    if (set == null || set.isEmpty()) {
+    if (set == null) {
       return "";
     }
+    String sep = "";
     StringBuilder valueStr = new StringBuilder();
     Iterator<E> it = set.iterator();
-    for (int i = 0; i < (set.size() - 1); i++) {
-      valueStr.append(it.next().getName());
-      valueStr.append(",");
+    while (it.hasNext()) {
+      valueStr.append(sep).append(it.next().getName());
+      sep = ",";
     }
-    valueStr.append(it.next().getName());
     return valueStr.toString();
   }
 
@@ -457,4 +457,16 @@ public class MetastoreUtil {
       cols.add(dim.getName().toLowerCase());
     }
   }
+
+  public static String getPartitionInfoKeyPrefix(UpdatePeriod updatePeriod, String partCol) {
+    return STORAGE_PFX + PARTITION_TIMELINE_CACHE + updatePeriod.getName() + "." + partCol + ".";
+  }
+
+  public static String getPartitionTimelineStorageClassKey(UpdatePeriod updatePeriod, String partCol) {
+    return getPartitionInfoKeyPrefix(updatePeriod, partCol) + STORAGE_CLASS;
+  }
+
+  public static String getPartitoinTimelineCachePresenceKey() {
+    return STORAGE_PFX + PARTITION_TIMELINE_CACHE + "present";
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
index 240516d..f1692b2 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
@@ -326,35 +326,38 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
       String dbName = SessionState.get().getCurrentDatabase();
       Table storageTbl = client.getTable(storageTableName);
       // update latest info
-      for (Map.Entry<String, LatestInfo> entry : updateLatestInfo.entrySet()) {
-        String latestPartCol = entry.getKey();
-        // symlink this partition to latest
-        List<Partition> latestParts;
-        try {
-          latestParts = client.getPartitionsByFilter(storageTbl, StorageConstants.getLatestPartFilter(latestPartCol));
-        } catch (Exception e) {
-          throw new HiveException("Could not get latest partition", e);
-        }
-        if (!latestParts.isEmpty()) {
-          client.dropPartition(storageTbl.getTableName(), latestParts.get(0).getValues(), false);
-        }
-        LatestInfo latest = entry.getValue();
-        if (latest != null && latest.part != null) {
-          AddPartitionDesc latestPart = new AddPartitionDesc(dbName, storageTableName, true);
-          latestPart.addPartition(StorageConstants.getLatestPartSpec(latest.part.getSpec(), latestPartCol),
-            latest.part.getLocation());
-          latestPart.getPartition(0).setPartParams(
-            latest.latestParts.get(latestPartCol).getPartParams(latest.part.getParameters()));
-          latestPart.getPartition(0).setInputFormat(latest.part.getInputFormatClass().getCanonicalName());
-          latestPart.getPartition(0).setOutputFormat(latest.part.getOutputFormatClass().getCanonicalName());
-          latestPart.getPartition(0).setNumBuckets(latest.part.getBucketCount());
-          latestPart.getPartition(0).setCols(latest.part.getCols());
-          latestPart.getPartition(0).setSerializationLib(
-            latest.part.getTPartition().getSd().getSerdeInfo().getSerializationLib());
-          latestPart.getPartition(0).setSerdeParams(latest.part.getTPartition().getSd().getSerdeInfo().getParameters());
-          latestPart.getPartition(0).setBucketCols(latest.part.getBucketCols());
-          latestPart.getPartition(0).setSortCols(latest.part.getSortCols());
-          client.createPartitions(latestPart);
+      if (updateLatestInfo != null) {
+        for (Map.Entry<String, LatestInfo> entry : updateLatestInfo.entrySet()) {
+          String latestPartCol = entry.getKey();
+          // symlink this partition to latest
+          List<Partition> latestParts;
+          try {
+            latestParts = client.getPartitionsByFilter(storageTbl, StorageConstants.getLatestPartFilter(latestPartCol));
+          } catch (Exception e) {
+            throw new HiveException("Could not get latest partition", e);
+          }
+          if (!latestParts.isEmpty()) {
+            client.dropPartition(storageTbl.getTableName(), latestParts.get(0).getValues(), false);
+          }
+          LatestInfo latest = entry.getValue();
+          if (latest != null && latest.part != null) {
+            AddPartitionDesc latestPart = new AddPartitionDesc(dbName, storageTableName, true);
+            latestPart.addPartition(StorageConstants.getLatestPartSpec(latest.part.getSpec(), latestPartCol),
+              latest.part.getLocation());
+            latestPart.getPartition(0).setPartParams(
+              latest.latestParts.get(latestPartCol).getPartParams(latest.part.getParameters()));
+            latestPart.getPartition(0).setInputFormat(latest.part.getInputFormatClass().getCanonicalName());
+            latestPart.getPartition(0).setOutputFormat(latest.part.getOutputFormatClass().getCanonicalName());
+            latestPart.getPartition(0).setNumBuckets(latest.part.getBucketCount());
+            latestPart.getPartition(0).setCols(latest.part.getCols());
+            latestPart.getPartition(0).setSerializationLib(
+              latest.part.getTPartition().getSd().getSerdeInfo().getSerializationLib());
+            latestPart.getPartition(0).setSerdeParams(
+              latest.part.getTPartition().getSd().getSerdeInfo().getParameters());
+            latestPart.getPartition(0).setBucketCols(latest.part.getBucketCols());
+            latestPart.getPartition(0).setSortCols(latest.part.getSortCols());
+            client.createPartitions(latestPart);
+          }
         }
       }
       commitDropPartition(storageTableName, partVals);

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
index c4b3c3a..aee9f7b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
@@ -43,6 +43,10 @@ public final class StorageConstants {
     return partCol + "='" + LATEST_PARTITION_VALUE + "'";
   }
 
+  public static String getPartFilter(String partCol, String value) {
+    return partCol + "='" + value + "'";
+  }
+
   /**
    * Get the latest partition value as List
    *

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
new file mode 100644
index 0000000..6a5b31d
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import java.text.ParseException;
+import java.util.Calendar;
+import java.util.Date;
+
+import org.apache.lens.api.LensException;
+
+import org.apache.commons.lang3.time.DateUtils;
+
+import lombok.Data;
+import lombok.NonNull;
+
+/** stores a partition's update period, date and string representation. Provides some utility methods around it */
+@Data
+public class TimePartition implements Comparable<TimePartition> {
+  private static final String UPDATE_PERIOD_WRONG_ERROR_MESSAGE = "Update period %s not correct for parsing %s";
+  private final UpdatePeriod updatePeriod;
+  private final Date date;
+  private final String dateString;
+
+  private TimePartition(@NonNull UpdatePeriod updatePeriod, @NonNull Date date) {
+    this.updatePeriod = updatePeriod;
+    Calendar cal = Calendar.getInstance();
+    cal.setTime(date);
+    this.date = truncate(date, updatePeriod);
+    this.dateString = updatePeriod.format().format(this.date);
+  }
+
+  private TimePartition(@NonNull UpdatePeriod updatePeriod, @NonNull String dateString) throws LensException {
+    this.updatePeriod = updatePeriod;
+    this.dateString = dateString;
+    try {
+      this.date = updatePeriod.format().parse(dateString);
+    } catch (ParseException e) {
+      throw new LensException(getWrongUpdatePeriodMessage(updatePeriod, dateString), e);
+    }
+    if (!updatePeriod.format().format(this.date).equals(this.dateString)) {
+      throw new LensException(getWrongUpdatePeriodMessage(updatePeriod, dateString));
+    }
+  }
+
+  public static TimePartition of(UpdatePeriod updatePeriod, Date date) throws LensException {
+    if (date == null) {
+      throw new LensException("time parition date is null");
+    }
+    return new TimePartition(updatePeriod, date);
+  }
+
+  public static TimePartition of(UpdatePeriod updatePeriod, String dateString) throws LensException {
+    if (dateString == null || dateString.isEmpty()) {
+      throw new LensException("time parition date string is null or blank");
+    } else {
+      return new TimePartition(updatePeriod, dateString);
+    }
+  }
+
+  public String toString() {
+    return dateString;
+  }
+
+  @Override
+  public int compareTo(TimePartition o) {
+    if (o == null) {
+      return 1;
+    }
+    return this.date.compareTo(o.date);
+  }
+
+  public TimePartition partitionAtDiff(int increment) {
+    Calendar cal = Calendar.getInstance();
+    cal.setTime(date);
+    cal.add(updatePeriod.calendarField(), increment);
+    return new TimePartition(updatePeriod, cal.getTime());
+  }
+
+  public TimePartition previous() {
+    return partitionAtDiff(-1);
+  }
+
+  public TimePartition next() {
+    return partitionAtDiff(1);
+  }
+
+  public boolean before(TimePartition when) {
+    return this.date.before(when.date);
+  }
+
+  public boolean after(TimePartition when) {
+    return this.date.after(when.date);
+  }
+
+  private Date truncate(Date date, UpdatePeriod updatePeriod) {
+    if (updatePeriod.equals(UpdatePeriod.WEEKLY)) {
+      Date truncDate = DateUtils.truncate(date, Calendar.DAY_OF_MONTH);
+      Calendar cal = Calendar.getInstance();
+      cal.setTime(truncDate);
+      cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
+      return cal.getTime();
+    } else {
+      return DateUtils.truncate(date, updatePeriod.calendarField());
+    }
+  }
+
+  protected static String getWrongUpdatePeriodMessage(UpdatePeriod up, String dateString) {
+    return String.format(UPDATE_PERIOD_WRONG_ERROR_MESSAGE, up, dateString);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
index f48acf2..e349f58 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
@@ -31,7 +31,7 @@ public enum UpdatePeriod implements Named {
   SECONDLY(Calendar.SECOND, 1000, "yyyy-MM-dd-HH-mm-ss"), MINUTELY(Calendar.MINUTE, 60 * SECONDLY.weight(),
     "yyyy-MM-dd-HH-mm"), HOURLY(Calendar.HOUR_OF_DAY, 60 * MINUTELY.weight(), "yyyy-MM-dd-HH"), DAILY(
     Calendar.DAY_OF_MONTH, 24 * HOURLY.weight(), "yyyy-MM-dd"), WEEKLY(Calendar.WEEK_OF_YEAR, 7 * DAILY.weight(),
-    "YYYY-'W'ww"), MONTHLY(Calendar.MONTH, 30 * DAILY.weight(), "yyyy-MM"), QUARTERLY(Calendar.MONTH, 3 * MONTHLY
+    "yyyy-'W'ww"), MONTHLY(Calendar.MONTH, 30 * DAILY.weight(), "yyyy-MM"), QUARTERLY(Calendar.MONTH, 3 * MONTHLY
     .weight(), "yyyy-MM"), YEARLY(Calendar.YEAR, 12 * MONTHLY.weight(), "yyyy");
 
   public static final long MIN_INTERVAL = SECONDLY.weight();

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
new file mode 100644
index 0000000..57dae9e
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata.timeline;
+
+
+import java.util.*;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.cube.metadata.CubeMetastoreClient;
+import org.apache.lens.cube.metadata.TimePartition;
+import org.apache.lens.cube.metadata.UpdatePeriod;
+import org.apache.lens.cube.parse.TimeRange;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import lombok.Data;
+import lombok.NonNull;
+
+/**
+ * One implementation of PartitionTimeline that stores first partition, latest partition and a collection of holes in
+ * between them, excluding the edges(start and end values).  This is the default Timeline for tables that don't specify
+ * which Timeline Class to use in it's params.
+ */
+@Data
+public class EndsAndHolesPartitionTimeline extends PartitionTimeline {
+  private TimePartition first;
+  private TreeSet<TimePartition> holes = new TreeSet<TimePartition>();
+  private TimePartition latest;
+
+  public EndsAndHolesPartitionTimeline(CubeMetastoreClient client, String storageTableName, UpdatePeriod updatePeriod,
+    String partCol) {
+    super(client, storageTableName, updatePeriod, partCol);
+  }
+
+  @Override
+  public boolean add(TimePartition partition) throws LensException {
+    if (isEmpty()) {
+      // First partition being added
+      first = partition;
+      latest = partition;
+      return true;
+    }
+    if (partition.before(first)) {
+      addHolesBetween(partition, first, partition.getUpdatePeriod());
+      first = partition;
+      return true;
+    } else if (partition.after(latest)) {
+      addHolesBetween(latest, partition, partition.getUpdatePeriod());
+      latest = partition;
+      return true;
+    } else {
+      return holes.remove(partition);
+    }
+  }
+
+  @Override
+  public boolean add(@NonNull Collection<TimePartition> partitions) throws LensException {
+    boolean result = true;
+    for (TimePartition partition : partitions) {
+      result &= add(partition);
+    }
+    // Can also return the failed to add items.
+    return result;
+  }
+
+  @Override
+  public boolean drop(TimePartition toDrop) throws LensException {
+    if (morePartitionsExist(toDrop.getDateString())) {
+      return true;
+    }
+    if (first.equals(latest) && first.equals(toDrop)) {
+      this.first = null;
+      this.latest = null;
+      this.holes.clear();
+      return true;
+    } else if (first.equals(toDrop)) {
+      this.first = this.getNextPartition(first, latest, 1);
+      return true;
+    } else if (latest.equals(toDrop)) {
+      this.latest = this.getNextPartition(latest, first, -1);
+      return true;
+    } else {
+      return addHole(toDrop);
+    }
+  }
+
+  @Override
+  public TimePartition latest() {
+    return latest;
+  }
+
+  @Override
+  public Map<String, String> toProperties() {
+    HashMap<String, String> ret = Maps.newHashMap();
+    if (isEmpty()) {
+      return ret;
+    }
+    ret.put("first", first.getDateString());
+    ret.put("latest", latest.getDateString());
+    ret.put("holes", StringUtils.join(holes, ","));
+    return ret;
+  }
+
+  @Override
+  public boolean initFromProperties(Map<String, String> properties) throws LensException {
+    first = TimePartition.of(getUpdatePeriod(), properties.get("first"));
+    latest = TimePartition.of(getUpdatePeriod(), properties.get("latest"));
+    holes = Sets.newTreeSet();
+    String holesStr = properties.get("holes");
+    if (!Strings.isNullOrEmpty(holesStr)) {
+      for (String hole : properties.get("holes").split("\\s*,\\s*")) {
+        holes.add(TimePartition.of(getUpdatePeriod(), hole));
+      }
+    }
+    return isConsistent();
+  }
+
+  private boolean addHole(TimePartition toDrop) {
+    return holes.add(toDrop);
+  }
+
+  private void addHolesBetween(TimePartition begin, TimePartition end, UpdatePeriod updatePeriod) throws LensException {
+    for (Date date : TimeRange.iterable(begin.next().getDate(), end.getDate(), updatePeriod, 1)) {
+      addHole(TimePartition.of(updatePeriod, date));
+    }
+  }
+
+  private TimePartition getNextPartition(TimePartition begin, TimePartition end, int increment) throws LensException {
+    for (Date date : TimeRange.iterable(begin.partitionAtDiff(increment).getDate(),
+      end.partitionAtDiff(increment).getDate(), begin.getUpdatePeriod(), increment)) {
+      TimePartition value = TimePartition.of(begin.getUpdatePeriod(), date);
+      if (!holes.contains(value)) {
+        return value;
+      } else {
+        holes.remove(value);
+      }
+    }
+    return null;
+  }
+
+
+  public boolean isEmpty() {
+    return first == null && latest == null && holes.isEmpty();
+  }
+
+  @Override
+  public boolean isConsistent() {
+    if (first == null && latest != null) {
+      return false;
+    }
+    if (latest == null && first != null) {
+      return false;
+    }
+    for (TimePartition hole : holes) {
+      if (!hole.after(first) || !hole.before(latest)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  @Override
+  public boolean exists(TimePartition toCheck) {
+    return !isEmpty() && !toCheck.before(first) && !toCheck.after(latest) && !holes.contains(toCheck);
+  }
+
+  @Override
+  public Iterator<TimePartition> iterator() {
+
+    return new Iterator<TimePartition>() {
+      TimePartition cur = getFirst();
+
+      @Override
+      public boolean hasNext() {
+        return cur != null && getLatest() != null && !cur.after(getLatest());
+      }
+
+      @Override
+      public TimePartition next() {
+        while (holes.contains(cur)) {
+          cur = cur.next();
+        }
+        TimePartition toReturn = cur;
+        cur = cur.next();
+        return toReturn;
+      }
+
+      @Override
+      public void remove() {
+        throw new UnsupportedOperationException();
+      }
+    };
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
new file mode 100644
index 0000000..64771d0
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
@@ -0,0 +1,207 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata.timeline;
+
+
+import java.util.*;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.cube.metadata.*;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import lombok.Data;
+import lombok.NonNull;
+
+/**
+ * Represents the in-memory data structure that represents timeline of all existing partitions for a given storage
+ * table, update period, partition column. Is an Abstract class. Can be implemented in multiple ways.
+ *
+ * @see org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline
+ * @see org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline
+ */
+@Data
+public abstract class PartitionTimeline implements Iterable<TimePartition> {
+  private final CubeMetastoreClient client;
+  private final String storageTableName;
+  private final UpdatePeriod updatePeriod;
+  private final String partCol;
+  private TreeSet<TimePartition> all;
+
+  /** wrapper on latest data */
+  public Date getLatestDate() {
+    return latest() == null ? null : latest().getDate();
+  }
+
+  /**
+   * Sets PartitionTimeline implementation class's name and specific params in table param.
+   *
+   * @param table
+   * @see #init(org.apache.hadoop.hive.ql.metadata.Table)
+   */
+  public void updateTableParams(Table table) {
+    String prefix = MetastoreUtil.getPartitionInfoKeyPrefix(getUpdatePeriod(), getPartCol());
+    String storageClass = MetastoreUtil.getPartitionTimelineStorageClassKey(getUpdatePeriod(), getPartCol());
+    table.getParameters().put(storageClass, this.getClass().getCanonicalName());
+    for (Map.Entry<String, String> entry : toProperties().entrySet()) {
+      table.getParameters().put(prefix + entry
+        .getKey(), entry.getValue());
+    }
+  }
+
+  /**
+   * Extracts timeline implementation class from table params and instantiates it with other arguments, also in table
+   * params.
+   *
+   * @param table
+   * @throws LensException
+   * @see #updateTableParams(org.apache.hadoop.hive.ql.metadata.Table)
+   */
+  public void init(Table table) throws LensException {
+    HashMap<String, String> props = Maps.newHashMap();
+    String prefix = MetastoreUtil.getPartitionInfoKeyPrefix(getUpdatePeriod(), getPartCol());
+    for (Map.Entry<String, String> entry : table.getParameters().entrySet()) {
+      if (entry.getKey().startsWith(prefix)) {
+        props.put(entry.getKey().substring(prefix.length()), entry.getValue());
+      }
+    }
+    initFromProperties(props);
+  }
+
+  /**
+   * Add partition to local memory to be sent for batch addition.
+   *
+   * @see #commitBatchAdditions()
+   */
+  public void addForBatchAddition(TimePartition partition) {
+    if (all == null) {
+      all = Sets.newTreeSet();
+    }
+    all.add(partition);
+  }
+
+  /**
+   * Commit all partitions that were added to batch addition queue. //TODO: improve batch addition implementation.
+   *
+   * @return true if all the partitions were added successfully, or no partitions needed to be added
+   * @throws LensException
+   */
+  public boolean commitBatchAdditions() throws LensException {
+    if (getAll() == null) {
+      return true;
+    }
+    boolean result = add(getAll());
+    all = null;
+    return result;
+  }
+
+  /**
+   * goes to metastore and queries if more partitions exist associated with (partCol = value) in storage table
+   * #getStorageTableName for update period #getUpdatePeriod. This might be useful for implementations while
+   * implementing drop.
+   *
+   * @param value
+   * @return
+   * @throws LensException
+   */
+  public boolean morePartitionsExist(String value) throws LensException {
+    try {
+      return getClient().partitionExistsByFilter(getStorageTableName(), StorageConstants.getPartFilter(getPartCol(),
+        value));
+    } catch (HiveException e) {
+      throw new LensException(e);
+    }
+  }
+
+  /**
+   * Add partition to timeline
+   *
+   * @param partition
+   * @return whether add was successful
+   * @throws LensException
+   */
+  public abstract boolean add(@NonNull TimePartition partition) throws LensException;
+
+  /**
+   * Add multiple partitions to timeline
+   *
+   * @param partitions
+   * @return whether add was successful
+   * @throws LensException
+   */
+  public abstract boolean add(@NonNull Collection<TimePartition> partitions) throws LensException;
+
+  /**
+   * drop partition.
+   *
+   * @param toDrop
+   * @return whether drop was successful
+   * @throws LensException
+   */
+  public abstract boolean drop(@NonNull TimePartition toDrop) throws LensException;
+
+  /**
+   * latest partition. will be null if no partitions exist.
+   *
+   * @return
+   */
+  public abstract TimePartition latest();
+
+  /**
+   * serialize member objects as map
+   *
+   * @return
+   */
+  public abstract Map<String, String> toProperties();
+
+  /**
+   * deserialize member variables from given map
+   *
+   * @param properties
+   * @return true if after deserializing, the timeline is in consistent state
+   * @throws LensException
+   * @see #isConsistent()
+   */
+  public abstract boolean initFromProperties(Map<String, String> properties) throws LensException;
+
+  /**
+   * Whether No partitions have been registered
+   *
+   * @return
+   */
+  public abstract boolean isEmpty();
+
+  /**
+   * whether timeline is in consistent state
+   *
+   * @return
+   */
+  public abstract boolean isConsistent();
+
+  /**
+   * Checks partition existance
+   *
+   * @param partition
+   * @return
+   */
+  public abstract boolean exists(TimePartition partition);
+}

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java
new file mode 100644
index 0000000..5626a03
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata.timeline;
+
+
+import java.lang.reflect.Constructor;
+
+import org.apache.lens.cube.metadata.CubeMetastoreClient;
+import org.apache.lens.cube.metadata.MetastoreUtil;
+import org.apache.lens.cube.metadata.UpdatePeriod;
+
+/** factory class for getting new timeline instances */
+public final class PartitionTimelineFactory {
+  private PartitionTimelineFactory() {
+
+  }
+
+  /**
+   * Checks in table params if desired implementing class is given. Otherwise by default returns instance of {@link
+   * org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline}.
+   *
+   * @param client
+   * @param storageTable
+   * @param updatePeriod
+   * @param partitionColumn
+   * @return
+   */
+  public static PartitionTimeline get(CubeMetastoreClient client, String storageTable,
+    UpdatePeriod updatePeriod, String partitionColumn) {
+    try {
+      String storageClassName = client.getTable(storageTable).getParameters().get(
+        MetastoreUtil.getPartitionTimelineStorageClassKey(
+          updatePeriod, partitionColumn));
+      Class<? extends PartitionTimeline> clz = (Class<? extends PartitionTimeline>) Class.forName(storageClassName);
+      Constructor<? extends PartitionTimeline> constructor = clz.getConstructor(
+        CubeMetastoreClient.class, String.class, UpdatePeriod.class, String.class);
+      return constructor.newInstance(
+        client, storageTable, updatePeriod, partitionColumn);
+    } catch (Exception e) {
+      return new EndsAndHolesPartitionTimeline(client, storageTable, updatePeriod, partitionColumn);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
new file mode 100644
index 0000000..8a9450c
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata.timeline;
+
+import java.util.*;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.cube.metadata.CubeMetastoreClient;
+import org.apache.lens.cube.metadata.TimePartition;
+import org.apache.lens.cube.metadata.UpdatePeriod;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import lombok.Data;
+import lombok.NonNull;
+
+/**
+ * Implementation of PartitionTimeline that stores all partitions as a tree set. Created for instrumentation purposes.
+ * Helps in test cases. In real life scenarios, this would be almost always end up causing out of memory error.
+ */
+@Data
+public class StoreAllPartitionTimeline extends PartitionTimeline {
+  TreeSet<TimePartition> allPartitions;
+
+  public StoreAllPartitionTimeline(CubeMetastoreClient client, String storageTableName,
+    UpdatePeriod updatePeriod, String partCol) {
+    super(client, storageTableName, updatePeriod, partCol);
+    allPartitions = Sets.newTreeSet();
+  }
+
+  @Override
+  public boolean add(@NonNull TimePartition partition) throws LensException {
+    return allPartitions.add(partition);
+  }
+
+  @Override
+  public boolean add(@NonNull Collection<TimePartition> partitions) throws LensException {
+    return allPartitions.addAll(partitions);
+  }
+
+  @Override
+  public boolean drop(@NonNull TimePartition toDrop) throws LensException {
+    if (morePartitionsExist(toDrop.getDateString())) {
+      return true;
+    }
+    return allPartitions.remove(toDrop);
+  }
+
+  @Override
+  public TimePartition latest() {
+    return allPartitions.size() == 0 ? null : allPartitions.last();
+  }
+
+  @Override
+  public Map<String, String> toProperties() {
+    HashMap<String, String> map = Maps.newHashMap();
+    map.put("partitions", StringUtils.join(allPartitions, ","));
+    return map;
+  }
+
+  @Override
+  public boolean initFromProperties(Map<String, String> properties) throws LensException {
+    String partitionsStr = properties.get("partitions");
+    if (partitionsStr == null) {
+      return true;
+    }
+    boolean ret = true;
+    for (String s : StringUtils.split(partitionsStr, ",")) {
+      ret &= add(TimePartition.of(getUpdatePeriod(), s));
+    }
+    return ret;
+  }
+
+  @Override
+  public boolean isEmpty() {
+    return allPartitions.isEmpty();
+  }
+
+  @Override
+  public boolean isConsistent() {
+    return true;
+  }
+
+  @Override
+  public boolean exists(TimePartition partition) {
+    return allPartitions.contains(partition);
+  }
+
+  @Override
+  public Iterator<TimePartition> iterator() {
+    return getAllPartitions().iterator();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java
index b1a85ad..d5276e0 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java
@@ -21,13 +21,20 @@ package org.apache.lens.cube.parse;
 
 import java.util.*;
 
+import org.apache.lens.cube.metadata.FactPartition;
+
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
 /**
  * Collapses the time range filters using IN operators
  */
 public class AbridgedTimeRangeWriter implements TimeRangeWriter {
+  //TODO: minimize use of String, use StringBuilders
+
   /**
    * Return IN clause for the partitions selected in the cube query
    *
@@ -44,10 +51,29 @@ public class AbridgedTimeRangeWriter implements TimeRangeWriter {
     if (parts == null || parts.isEmpty()) {
       return "";
     }
-
     // Collect partition specs by column in a map
     // All filters which contain only a single column will be combined in an IN operator clause
     // This clause will be ORed with filters which contain multiple columns.
+    List<String> subFilters = new ArrayList<String>();
+    for (Map.Entry<Set<FactPartition>, Set<FactPartition>> entry : groupPartitions(parts).entrySet()) {
+      List<String> clauses = new ArrayList<String>();
+      String clause;
+      clause = getClause(cubeQueryContext, tableName, entry.getKey());
+      if (clause != null && !clause.isEmpty()) {
+        clauses.add(clause);
+      }
+      clause = getClause(cubeQueryContext, tableName, entry.getValue());
+      if (clause != null && !clause.isEmpty()) {
+        clauses.add(clause);
+      }
+      subFilters.add("(" + StringUtils.join(clauses, " AND ") + ")");
+    }
+    return StringUtils.join(subFilters, " OR ");
+  }
+
+  private String getClause(CubeQueryContext cubeQueryContext,
+    String tableName,
+    Set<FactPartition> parts) throws SemanticException {
     Map<String, List<String>> partFilterMap = new HashMap<String, List<String>>();
     List<String> allTimeRangeFilters = new ArrayList<String>();
 
@@ -71,6 +97,53 @@ public class AbridgedTimeRangeWriter implements TimeRangeWriter {
     return StringUtils.join(allTimeRangeFilters, " OR ");
   }
 
+  /**
+   * parts is a collection of FactPartition objects. And FactPartition can be viewed as two boolean conditions, one
+   * specified by it's containingPart object, and another specified by itself in the form (partCol = partSpec)
+   * <p/>
+   * Collection of FactPartition objects can be viewed as an OR clause on all the FactPartition objects -- which by
+   * itself is a binary AND clause.
+   * <p/>
+   * So Collection&lt;FactPartition&gt; is nothing but (a AND b) OR (c AND d) OR (e AND f) ...
+   * <p/>
+   * This function tries to reduce such a big clause by using Boolean arithmetic. The big thing it aims to reduce is the
+   * following class of clauses:
+   * <p/>
+   * (a AND c) OR (a AND d) OR (b AND c) OR (b AND d) => ((a OR b) AND (c OR d))
+   * <p/>
+   * Equivalent return value for such a reduction would be an entry in the returned map from set(a,b) to set(c,d).
+   * Assuming the argument was set(a(containing=c), a(containing=d), b(containing=c), b(containing=d))
+   *
+   * @param parts
+   * @return
+   */
+  private Map<Set<FactPartition>, Set<FactPartition>> groupPartitions(Collection<FactPartition> parts) {
+    Map<FactPartition, Set<FactPartition>> partitionSetMap = new HashMap<FactPartition, Set<FactPartition>>();
+    for (FactPartition part : parts) {
+      FactPartition key = part.getContainingPart();
+      part.setContainingPart(null);
+      if (partitionSetMap.get(key) == null) {
+        partitionSetMap.put(key, Sets.<FactPartition>newTreeSet());
+      }
+      partitionSetMap.get(key).add(part);
+    }
+    Map<Set<FactPartition>, Set<FactPartition>> setSetOppositeMap = Maps.newHashMap();
+    for (Map.Entry<FactPartition, Set<FactPartition>> entry : partitionSetMap.entrySet()) {
+      if (setSetOppositeMap.get(entry.getValue()) == null) {
+        setSetOppositeMap.put(entry.getValue(), Sets.<FactPartition>newTreeSet());
+      }
+      if (entry.getKey() != null) {
+        setSetOppositeMap.get(entry.getValue()).add(entry.getKey());
+      }
+    }
+
+    Map<Set<FactPartition>, Set<FactPartition>> setSetMap = Maps.newHashMap();
+    for (Map.Entry<Set<FactPartition>, Set<FactPartition>> entry : setSetOppositeMap.entrySet()) {
+      setSetMap.put(entry.getValue(), entry.getKey());
+    }
+    return setSetMap;
+  }
+
   // This takes the output of filter generated by TimeRangeUtils.getTimeRangePartitionFilter
   // splits the filters by column names and filters are collected by column name in the
   // map passed as argument

http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/803448a1/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
index 14ea806..6c85c2d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
@@ -22,6 +22,8 @@ import java.util.Iterator;
 import java.util.Set;
 import java.util.TreeSet;
 
+import org.apache.lens.cube.metadata.FactPartition;
+
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.parse.SemanticException;