You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/04/15 22:17:42 UTC
incubator-lens git commit: LENS-499 : Allow multiple 'latest'
partitions for different combination of non time partition values (Rajat
Khandelwal via amareshwari)
Repository: incubator-lens
Updated Branches:
refs/heads/master b10d15be5 -> 120bf6cd7
LENS-499 : Allow multiple 'latest' partitions for different combination of non time partition values (Rajat Khandelwal via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/120bf6cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/120bf6cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/120bf6cd
Branch: refs/heads/master
Commit: 120bf6cd7e13949d80aef5fff8ed5ad9b9dd1128
Parents: b10d15b
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Wed Apr 15 15:17:35 2015 -0500
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Apr 15 15:17:35 2015 -0500
----------------------------------------------------------------------
checkstyle/src/main/resources/checkstyle.xml | 2 +-
.../lens/cube/metadata/CubeMetastoreClient.java | 80 +++++----
.../lens/cube/metadata/MetastoreConstants.java | 2 +
.../lens/cube/metadata/MetastoreUtil.java | 50 +++++-
.../org/apache/lens/cube/metadata/Storage.java | 101 ++++++-----
.../lens/cube/metadata/StorageConstants.java | 35 +++-
.../cube/metadata/TestCubeMetastoreClient.java | 171 ++++++++++++++++---
.../apache/lens/cube/parse/CubeTestSetup.java | 2 +-
8 files changed, 340 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/checkstyle/src/main/resources/checkstyle.xml
----------------------------------------------------------------------
diff --git a/checkstyle/src/main/resources/checkstyle.xml b/checkstyle/src/main/resources/checkstyle.xml
index a663f9c..236f977 100644
--- a/checkstyle/src/main/resources/checkstyle.xml
+++ b/checkstyle/src/main/resources/checkstyle.xml
@@ -60,7 +60,7 @@
<module name="NewlineAtEndOfFile"/>
<module name="FileLength">
- <property name="max" value="2500"/>
+ <property name="max" value="3500"/>
</module>
<module name="FileTabCharacter"/>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 324002f..fbdefab 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -188,7 +188,7 @@ public class CubeMetastoreClient {
if (get(storageTableName) == null) {
Table storageTable = getTable(storageTableName);
if ("true".equalsIgnoreCase(storageTable.getParameters().get(
- MetastoreUtil.getPartitoinTimelineCachePresenceKey()))) {
+ MetastoreUtil.getPartitionTimelineCachePresenceKey()))) {
try {
loadTimelinesFromTableProperties(fact, storage);
} catch (Exception e) {
@@ -700,8 +700,14 @@ public class CubeMetastoreClient {
Storage.getPrefix(storageName.trim())).toLowerCase();
if (getDimensionTable(factOrDimTable) != null) {
// Adding partition in dimension table.
+ Map<Map<String, String>, LatestInfo> latestInfos = Maps.newHashMap();
+ for (Map.Entry<Map<String, String>, List<StoragePartitionDesc>> entry : groupByNonTimePartitions(
+ storagePartitionDescs).entrySet()) {
+ latestInfos.put(entry.getKey(),
+ getDimTableLatestInfo(storageTableName, entry.getKey(), getTimePartSpecs(entry.getValue()), updatePeriod));
+ }
getStorage(storageName).addPartitions(getClient(), factOrDimTable, updatePeriod, storagePartitionDescs,
- getDimTableLatestInfo(storageTableName, getTimePartSpecs(storagePartitionDescs), updatePeriod));
+ latestInfos);
latestLookupCache.add(storageTableName);
} else {
// first update in memory, then add to hive table's partitions. delete is reverse.
@@ -759,7 +765,7 @@ public class CubeMetastoreClient {
entry.getValue().updateTableParams(table);
}
}
- params.put(MetastoreUtil.getPartitoinTimelineCachePresenceKey(), "true");
+ params.put(MetastoreUtil.getPartitionTimelineCachePresenceKey(), "true");
alterHiveTable(storageTableName, table);
}
}
@@ -769,7 +775,8 @@ public class CubeMetastoreClient {
return UpdatePeriod.valueOf(partition.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD));
}
- private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, TreeSet<Date>> partitionTimestamps,
+ private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, String> nonTimeParts,
+ Map<String, TreeSet<Date>> timePartSpecs,
UpdatePeriod updatePeriod) throws HiveException {
Table hiveTable = getHiveTable(storageTableName);
String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS);
@@ -777,13 +784,13 @@ public class CubeMetastoreClient {
LatestInfo latest = new LatestInfo();
String[] timePartCols = StringUtils.split(timePartColsStr, ',');
for (String partCol : timePartCols) {
- if (!partitionTimestamps.containsKey(partCol)) {
+ if (!timePartSpecs.containsKey(partCol)) {
continue;
}
boolean makeLatest = true;
- Partition part = getLatestPart(storageTableName, partCol);
- Date pTimestamp = partitionTimestamps.get(partCol).last();
- Date latestTimestamp = getLatestTimeStampOfDimtable(part, partCol);
+ Partition part = getLatestPart(storageTableName, partCol, nonTimeParts);
+ Date pTimestamp = timePartSpecs.get(partCol).last();
+ Date latestTimestamp = MetastoreUtil.getLatestTimeStampOfDimtable(part, partCol);
if (latestTimestamp != null && pTimestamp.before(latestTimestamp)) {
makeLatest = false;
}
@@ -800,22 +807,20 @@ public class CubeMetastoreClient {
}
}
- private boolean isLatestPartOfDimtable(Partition part) {
- return part.getValues().contains(StorageConstants.LATEST_PARTITION_VALUE);
- }
-
- public Date getLatestTimeStampOfDimtable(Partition part, String partCol) throws HiveException {
- if (part != null) {
- String latestTimeStampStr = part.getParameters().get(MetastoreUtil.getLatestPartTimestampKey(partCol));
- String latestPartUpdatePeriod = part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD);
- UpdatePeriod latestUpdatePeriod = UpdatePeriod.valueOf(latestPartUpdatePeriod.toUpperCase());
- try {
- return latestUpdatePeriod.format().parse(latestTimeStampStr);
- } catch (ParseException e) {
- throw new HiveException(e);
+ private Map<Map<String, String>, List<StoragePartitionDesc>> groupByNonTimePartitions(
+ List<StoragePartitionDesc> storagePartitionDescs) {
+ Map<Map<String, String>, List<StoragePartitionDesc>> result = Maps.newHashMap();
+ for (StoragePartitionDesc storagePartitionDesc : storagePartitionDescs) {
+ if (result.get(storagePartitionDesc.getNonTimePartSpec()) == null) {
+ result.put(storagePartitionDesc.getNonTimePartSpec(), Lists.<StoragePartitionDesc>newArrayList());
}
+ result.get(storagePartitionDesc.getNonTimePartSpec()).add(storagePartitionDesc);
}
- return null;
+ return result;
+ }
+
+ private boolean isLatestPartOfDimtable(Partition part) {
+ return part.getValues().contains(StorageConstants.LATEST_PARTITION_VALUE);
}
private Date getPartDate(Partition part, int timeColIndex) {
@@ -833,10 +838,17 @@ public class CubeMetastoreClient {
return partDate;
}
- private LatestInfo getNextLatestOfDimtable(Table hiveTable, String timeCol, final int timeColIndex)
+ private LatestInfo getNextLatestOfDimtable(Table hiveTable, String timeCol, final int timeColIndex,
+ UpdatePeriod updatePeriod, Map<String, String> nonTimePartSpec)
throws HiveException {
// getClient().getPartitionsByNames(tbl, partNames)
- List<Partition> partitions = getClient().getPartitions(hiveTable);
+ List<Partition> partitions = null;
+ try {
+ partitions = getClient().getPartitionsByFilter(hiveTable, StorageConstants.getPartFilter(nonTimePartSpec));
+ MetastoreUtil.filterPartitionsByNonTimeParts(partitions, nonTimePartSpec, timeCol);
+ } catch (TException e) {
+ throw new HiveException(e);
+ }
// tree set contains partitions with timestamp as value for timeCol, in
// descending order
@@ -867,7 +879,7 @@ public class CubeMetastoreClient {
}
}
Iterator<Partition> it = allPartTimeVals.iterator();
- it.next();
+ it.next(); // Skip itself. We have to find next latest.
LatestInfo latest = null;
if (it.hasNext()) {
Partition nextLatest = it.next();
@@ -920,7 +932,7 @@ public class CubeMetastoreClient {
throw new HiveException("Not a time partition column:" + timeCol);
}
int timeColIndex = partColNames.indexOf(timeCol);
- Partition part = getLatestPart(storageTableName, timeCol);
+ Partition part = getLatestPart(storageTableName, timeCol, nonTimePartSpec);
boolean isLatest = true;
// check if partition being dropped is the latest partition
@@ -933,7 +945,7 @@ public class CubeMetastoreClient {
}
}
if (isLatest) {
- Date latestTimestamp = getLatestTimeStampOfDimtable(part, timeCol);
+ Date latestTimestamp = MetastoreUtil.getLatestTimeStampOfDimtable(part, timeCol);
Date dropTimestamp;
try {
dropTimestamp = updatePeriod.format().parse(updatePeriod.format().format(timePartSpec.get(timeCol)));
@@ -941,7 +953,8 @@ public class CubeMetastoreClient {
throw new HiveException(e);
}
if (latestTimestamp != null && dropTimestamp.equals(latestTimestamp)) {
- LatestInfo latestInfo = getNextLatestOfDimtable(hiveTable, timeCol, timeColIndex);
+ LatestInfo latestInfo =
+ getNextLatestOfDimtable(hiveTable, timeCol, timeColIndex, updatePeriod, nonTimePartSpec);
latestAvailable = (latestInfo != null && latestInfo.part != null);
latest.put(timeCol, latestInfo);
}
@@ -954,14 +967,14 @@ public class CubeMetastoreClient {
throw new HiveException("Not time part columns" + timePartSpec.keySet());
}
}
- getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, latest);
+ getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, latest, nonTimePartSpec);
if (!latestAvailable) {
// dropping latest and could not find latest, removing the entry from latest lookup cache
latestLookupCache.remove(storageTableName);
}
} else {
// dropping fact partition
- getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, null);
+ getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, null, null);
if (partitionTimelineCache.updateForDeletion(cubeTableName, storageName, updatePeriod, timePartSpec)) {
this.alterTablePartitionCache(storageTableName);
}
@@ -1083,8 +1096,13 @@ public class CubeMetastoreClient {
}
Partition getLatestPart(String storageTableName, String latestPartCol) throws HiveException {
+ return getLatestPart(storageTableName, latestPartCol, null);
+ }
+
+ Partition getLatestPart(String storageTableName, String latestPartCol, Map<String, String> nonTimeParts)
+ throws HiveException {
List<Partition> latestParts =
- getPartitionsByFilter(storageTableName, StorageConstants.getLatestPartFilter(latestPartCol));
+ getPartitionsByFilter(storageTableName, StorageConstants.getLatestPartFilter(latestPartCol, nonTimeParts));
if (latestParts != null && !latestParts.isEmpty()) {
return latestParts.get(0);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
index e7f10ac..74a26fe 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
@@ -53,7 +53,9 @@ public final class MetastoreConstants {
public static final String FACT_AGGREGATED_PROPERTY = "cube.fact.is.aggregated";
// dim table constants
+ // TODO: remove this and move to "dimtable."
public static final String DIM_TBL_PFX = "dimtble.";
+ public static final String DIM_TABLE_PFX = "dimtable.";
public static final String DUMP_PERIOD_SFX = ".dumpperiod";
public static final String STORAGE_LIST_SFX = ".storages";
public static final String DIM_NAME_SFX = ".dim.name";
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index bdec4e3..8e2c9ed 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -21,10 +21,13 @@ package org.apache.lens.cube.metadata;
import static org.apache.lens.cube.metadata.MetastoreConstants.*;
+import java.text.ParseException;
import java.util.*;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
public class MetastoreUtil {
private MetastoreUtil() {
@@ -63,7 +66,7 @@ public class MetastoreUtil {
}
public static final String getDimTablePartsKey(String dimtableName) {
- return getDimensionTablePrefix(dimtableName) + PARTCOLS_SFX;
+ return DIM_TABLE_PFX + dimtableName + PARTCOLS_SFX;
}
public static final String getDimTimedDimensionKey(String dimName) {
@@ -401,7 +404,7 @@ public class MetastoreUtil {
}
static <E extends Named> void addNameStrings(Map<String, String> props, String key,
- Collection<E> set, int maxLength) {
+ Collection<E> set, int maxLength) {
List<String> namedStrings = getNamedStrs(set, maxLength);
props.put(key + ".size", String.valueOf(namedStrings.size()));
for (int i = 0; i < namedStrings.size(); i++) {
@@ -480,7 +483,48 @@ public class MetastoreUtil {
return getPartitionInfoKeyPrefix(updatePeriod, partCol) + STORAGE_CLASS;
}
- public static String getPartitoinTimelineCachePresenceKey() {
+ public static String getPartitionTimelineCachePresenceKey() {
return STORAGE_PFX + PARTITION_TIMELINE_CACHE + "present";
}
+
+ public static List<Partition> filterPartitionsByNonTimeParts(List<Partition> partitions,
+ Map<String, String> nonTimePartSpec,
+ String latestPartCol) {
+ ListIterator<Partition> iter = partitions.listIterator();
+ while (iter.hasNext()) {
+ Partition part = iter.next();
+ boolean ignore = false;
+ for (Map.Entry<String, String> entry1 : part.getSpec().entrySet()) {
+ if ((nonTimePartSpec == null || !nonTimePartSpec.containsKey(entry1.getKey()))
+ && !entry1.getKey().equals(latestPartCol)) {
+ try {
+ UpdatePeriod.valueOf(part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD))
+ .format()
+ .parse(entry1.getValue());
+ } catch (ParseException e) {
+ ignore = true;
+ }
+ }
+ }
+
+ if (ignore) {
+ iter.remove();
+ }
+ }
+ return partitions;
+ }
+
+ public static Date getLatestTimeStampOfDimtable(Partition part, String partCol) throws HiveException {
+ if (part != null) {
+ String latestTimeStampStr = part.getParameters().get(MetastoreUtil.getLatestPartTimestampKey(partCol));
+ String latestPartUpdatePeriod = part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD);
+ UpdatePeriod latestUpdatePeriod = UpdatePeriod.valueOf(latestPartUpdatePeriod.toUpperCase());
+ try {
+ return latestUpdatePeriod.format().parse(latestTimeStampStr);
+ } catch (ParseException e) {
+ throw new HiveException(e);
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
index 50ec4ce..d3c9974 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
@@ -237,25 +237,28 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
*/
public void addPartition(Hive client, StoragePartitionDesc addPartitionDesc, LatestInfo latestInfo)
throws HiveException {
- addPartitions(client, addPartitionDesc.getCubeTableName(), addPartitionDesc.getUpdatePeriod(), Arrays.asList(
- addPartitionDesc), latestInfo);
+ Map<Map<String, String>, LatestInfo> latestInfos = Maps.newHashMap();
+ latestInfos.put(addPartitionDesc.getNonTimePartSpec(), latestInfo);
+ addPartitions(client, addPartitionDesc.getCubeTableName(), addPartitionDesc.getUpdatePeriod(),
+ Collections.singletonList(addPartitionDesc), latestInfos);
}
/**
* Add given partitions in the underlying hive table and update latest partition links
*
* @param client hive client instance
- * @param fact fact name
+ * @param factOrDimTable fact or dim name
* @param updatePeriod update period of partitions.
* @param storagePartitionDescs all partitions to be added
- * @param latestInfo new latest info. atleast one partition for the latest value exists for each part
+ * @param latestInfos new latest info. atleast one partition for the latest value exists for each part
* column
* @throws HiveException
*/
public void addPartitions(Hive client, String factOrDimTable, UpdatePeriod updatePeriod,
- List<StoragePartitionDesc> storagePartitionDescs, LatestInfo latestInfo) throws HiveException {
+ List<StoragePartitionDesc> storagePartitionDescs,
+ Map<Map<String, String>, LatestInfo> latestInfos) throws HiveException {
preAddPartitions(storagePartitionDescs);
- Map<String, Integer> latestPartIndexForPartCols = Maps.newHashMap();
+ Map<Map<String, String>, Map<String, Integer>> latestPartIndexForPartCols = Maps.newHashMap();
boolean success = false;
try {
String tableName = MetastoreUtil.getStorageTableName(factOrDimTable, this.getPrefix());
@@ -288,46 +291,56 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
addParts.getPartition(curIndex).setSerdeParams(addPartitionDesc.getSerdeParams());
addParts.getPartition(curIndex).setBucketCols(addPartitionDesc.getBucketCols());
addParts.getPartition(curIndex).setSortCols(addPartitionDesc.getSortCols());
- if (latestInfo != null) {
- for (Map.Entry<String, LatestPartColumnInfo> entry : latestInfo.latestParts.entrySet()) {
+ if (latestInfos != null && latestInfos.get(addPartitionDesc.getNonTimePartSpec()) != null) {
+ for (Map.Entry<String, LatestPartColumnInfo> entry : latestInfos
+ .get(addPartitionDesc.getNonTimePartSpec()).latestParts.entrySet()) {
if (addPartitionDesc.getTimePartSpec().containsKey(entry.getKey())
&& entry.getValue().get(MetastoreUtil.getLatestPartTimestampKey(entry.getKey())).equals(
updatePeriod.format().format(addPartitionDesc.getTimePartSpec().get(entry.getKey())))) {
- latestPartIndexForPartCols.put(entry.getKey(), curIndex);
+ if (latestPartIndexForPartCols.get(addPartitionDesc.getNonTimePartSpec()) == null) {
+ latestPartIndexForPartCols.put(addPartitionDesc.getNonTimePartSpec(),
+ Maps.<String, Integer>newHashMap());
+ }
+ latestPartIndexForPartCols.get(addPartitionDesc.getNonTimePartSpec()).put(entry.getKey(), curIndex);
}
}
}
}
- if (latestInfo != null) {
- for (Map.Entry<String, LatestPartColumnInfo> entry : latestInfo.latestParts.entrySet()) {
- // symlink this partition to latest
- List<Partition> latest;
- String latestPartCol = entry.getKey();
- try {
- latest = client.getPartitionsByFilter(storageTbl, StorageConstants.getLatestPartFilter(latestPartCol));
- } catch (Exception e) {
- throw new HiveException("Could not get latest partition", e);
- }
- if (!latest.isEmpty()) {
- client.dropPartition(storageTbl.getTableName(), latest.get(0).getValues(), false);
- }
- if (latestPartIndexForPartCols.containsKey(latestPartCol)) {
- AddPartitionDesc.OnePartitionDesc latestPartWithFullTimestamp = addParts.getPartition(
- latestPartIndexForPartCols.get(latestPartCol));
- addParts.addPartition(
- StorageConstants.getLatestPartSpec(latestPartWithFullTimestamp.getPartSpec(), latestPartCol),
- latestPartWithFullTimestamp.getLocation());
- int curIndex = addParts.getPartitionCount() - 1;
- addParts.getPartition(curIndex).setPartParams(entry.getValue().getPartParams(
- latestPartWithFullTimestamp.getPartParams()));
- addParts.getPartition(curIndex).setInputFormat(latestPartWithFullTimestamp.getInputFormat());
- addParts.getPartition(curIndex).setOutputFormat(latestPartWithFullTimestamp.getOutputFormat());
- addParts.getPartition(curIndex).setNumBuckets(latestPartWithFullTimestamp.getNumBuckets());
- addParts.getPartition(curIndex).setCols(latestPartWithFullTimestamp.getCols());
- addParts.getPartition(curIndex).setSerializationLib(latestPartWithFullTimestamp.getSerializationLib());
- addParts.getPartition(curIndex).setSerdeParams(latestPartWithFullTimestamp.getSerdeParams());
- addParts.getPartition(curIndex).setBucketCols(latestPartWithFullTimestamp.getBucketCols());
- addParts.getPartition(curIndex).setSortCols(latestPartWithFullTimestamp.getSortCols());
+ if (latestInfos != null) {
+ for (Map.Entry<Map<String, String>, LatestInfo> entry1 : latestInfos.entrySet()) {
+ Map<String, String> nonTimeParts = entry1.getKey();
+ LatestInfo latestInfo = entry1.getValue();
+ for (Map.Entry<String, LatestPartColumnInfo> entry : latestInfo.latestParts.entrySet()) {
+ // symlink this partition to latest
+ List<Partition> latest;
+ String latestPartCol = entry.getKey();
+ try {
+ latest = client
+ .getPartitionsByFilter(storageTbl, StorageConstants.getLatestPartFilter(latestPartCol, nonTimeParts));
+ } catch (Exception e) {
+ throw new HiveException("Could not get latest partition", e);
+ }
+ if (!latest.isEmpty()) {
+ client.dropPartition(storageTbl.getTableName(), latest.get(0).getValues(), false);
+ }
+ if (latestPartIndexForPartCols.get(nonTimeParts).containsKey(latestPartCol)) {
+ AddPartitionDesc.OnePartitionDesc latestPartWithFullTimestamp = addParts.getPartition(
+ latestPartIndexForPartCols.get(nonTimeParts).get(latestPartCol));
+ addParts.addPartition(
+ StorageConstants.getLatestPartSpec(latestPartWithFullTimestamp.getPartSpec(), latestPartCol),
+ latestPartWithFullTimestamp.getLocation());
+ int curIndex = addParts.getPartitionCount() - 1;
+ addParts.getPartition(curIndex).setPartParams(entry.getValue().getPartParams(
+ latestPartWithFullTimestamp.getPartParams()));
+ addParts.getPartition(curIndex).setInputFormat(latestPartWithFullTimestamp.getInputFormat());
+ addParts.getPartition(curIndex).setOutputFormat(latestPartWithFullTimestamp.getOutputFormat());
+ addParts.getPartition(curIndex).setNumBuckets(latestPartWithFullTimestamp.getNumBuckets());
+ addParts.getPartition(curIndex).setCols(latestPartWithFullTimestamp.getCols());
+ addParts.getPartition(curIndex).setSerializationLib(latestPartWithFullTimestamp.getSerializationLib());
+ addParts.getPartition(curIndex).setSerdeParams(latestPartWithFullTimestamp.getSerdeParams());
+ addParts.getPartition(curIndex).setBucketCols(latestPartWithFullTimestamp.getBucketCols());
+ addParts.getPartition(curIndex).setSortCols(latestPartWithFullTimestamp.getSortCols());
+ }
}
}
}
@@ -348,10 +361,11 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
* @param storageTableName TableName
* @param partVals Partition specification
* @param updateLatestInfo The latest partition info if it needs update, null if latest should not be updated
+ * @param nonTimePartSpec
* @throws HiveException
*/
public void dropPartition(Hive client, String storageTableName, List<String> partVals,
- Map<String, LatestInfo> updateLatestInfo) throws HiveException {
+ Map<String, LatestInfo> updateLatestInfo, Map<String, String> nonTimePartSpec) throws HiveException {
preDropPartition(storageTableName, partVals);
boolean success = false;
try {
@@ -360,16 +374,19 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
Table storageTbl = client.getTable(storageTableName);
// update latest info
if (updateLatestInfo != null) {
- for (Map.Entry<String, LatestInfo> entry : updateLatestInfo.entrySet()) {
+ for (Entry<String, LatestInfo> entry : updateLatestInfo.entrySet()) {
String latestPartCol = entry.getKey();
// symlink this partition to latest
List<Partition> latestParts;
try {
- latestParts = client.getPartitionsByFilter(storageTbl, StorageConstants.getLatestPartFilter(latestPartCol));
+ latestParts = client.getPartitionsByFilter(storageTbl,
+ StorageConstants.getLatestPartFilter(latestPartCol, nonTimePartSpec));
+ MetastoreUtil.filterPartitionsByNonTimeParts(latestParts, nonTimePartSpec, latestPartCol);
} catch (Exception e) {
throw new HiveException("Could not get latest partition", e);
}
if (!latestParts.isEmpty()) {
+ assert latestParts.size() == 1;
client.dropPartition(storageTbl.getTableName(), latestParts.get(0).getValues(), false);
}
LatestInfo latest = entry.getValue();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
index 610d168..5881266 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
@@ -24,6 +24,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import com.google.common.collect.Maps;
+
public final class StorageConstants {
private StorageConstants() {
@@ -40,11 +42,38 @@ public final class StorageConstants {
* @return latest partition spec as Map from String to String
*/
public static String getLatestPartFilter(String partCol) {
- return partCol + "='" + LATEST_PARTITION_VALUE + "'";
+ return getPartFilter(partCol, LATEST_PARTITION_VALUE);
+ }
+
+ public static String getPartFilter(final String partCol, final String value) {
+ return getPartFilter(new HashMap<String, String>() {
+ {
+ put(partCol, value);
+ }
+ });
+ }
+
+ public static String getPartFilter(Map<String, String> parts) {
+ String sep = "";
+ StringBuilder ret = new StringBuilder();
+ if (parts != null) {
+ for (Map.Entry<String, String> entry : parts.entrySet()) {
+ ret.append(sep).append(entry.getKey()).append("='").append(entry.getValue()).append("'");
+ sep = " and ";
+ }
+ }
+ return ret.toString();
+ }
+
+ public static String getPartFilter(String partCol, String value, Map<String, String> parts) {
+ Map<String, String> allParts = Maps.newHashMap();
+ allParts.putAll(parts);
+ allParts.put(partCol, value);
+ return getPartFilter(allParts);
}
- public static String getPartFilter(String partCol, String value) {
- return partCol + "='" + value + "'";
+ public static String getLatestPartFilter(String partCol, Map<String, String> parts) {
+ return getPartFilter(partCol, LATEST_PARTITION_VALUE, parts);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 96f2b9b..d4c109a 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -47,6 +47,9 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
public class TestCubeMetastoreClient {
private static CubeMetastoreClient client;
@@ -67,6 +70,8 @@ public class TestCubeMetastoreClient {
private static final Map<String, String> CUBE_PROPERTIES = new HashMap<String, String>();
private static Date now;
private static Date nowPlus1;
+ private static Date nowPlus2;
+ private static Date nowPlus3;
private static Date nowMinus1;
private static Date nowMinus2;
private static Date nowMinus3;
@@ -114,6 +119,10 @@ public class TestCubeMetastoreClient {
cal.setTime(now);
cal.add(Calendar.HOUR_OF_DAY, 1);
nowPlus1 = cal.getTime();
+ cal.add(Calendar.HOUR_OF_DAY, 1);
+ nowPlus2 = cal.getTime();
+ cal.add(Calendar.HOUR_OF_DAY, 1);
+ nowPlus3 = cal.getTime();
cal.setTime(now);
cal.add(Calendar.HOUR, -1);
nowMinus1 = cal.getTime();
@@ -945,7 +954,7 @@ public class TestCubeMetastoreClient {
String c4TableName = MetastoreUtil.getFactStorageTableName(factName, c4);
Assert.assertTrue(client.tableExists(c4TableName));
- // add storage
+ // add storage
client.addStorage(altered2, c3, updates, s1);
CubeFactTable altered3 = client.getCubeFact(factName);
Assert.assertTrue(altered3.getStorages().contains("C3"));
@@ -1411,14 +1420,36 @@ public class TestCubeMetastoreClient {
}
}
- private TimePartition[] getLatestValues(String storageTableName, UpdatePeriod updatePeriod, String... partCols)
+ private TimePartition[] getLatestValues(String storageTableName, UpdatePeriod updatePeriod, String[] partCols,
+ Map<String, String> nonTimeParts)
throws LensException, HiveException {
TimePartition[] values = new TimePartition[partCols.length];
for (int i = 0; i < partCols.length; i++) {
- List<Partition> part = client.getPartitionsByFilter(storageTableName, partCols[i] + "='latest'");
- Assert.assertEquals(part.size(), 1);
- values[i] = TimePartition.of(updatePeriod, part.get(0).getParameters().get(
- MetastoreUtil.getLatestPartTimestampKey(partCols[i])));
+ List<Partition> parts = client.getPartitionsByFilter(storageTableName,
+ StorageConstants.getLatestPartFilter(partCols[i], nonTimeParts));
+ for (Partition part : parts) {
+ boolean ignore = false;
+ if (nonTimeParts != null) {
+ for (Map.Entry<String, String> entry : part.getSpec().entrySet()) {
+ if (!nonTimeParts.containsKey(entry.getKey())) {
+ try {
+ updatePeriod.format().parse(entry.getValue());
+ } catch (java.text.ParseException e) {
+ ignore = true;
+ break;
+ }
+ }
+ }
+ }
+ if (ignore) {
+ break;
+ }
+ TimePartition tp = TimePartition.of(updatePeriod, part.getParameters().get(
+ MetastoreUtil.getLatestPartTimestampKey(partCols[i])));
+ if (values[i] == null || values[i].before(tp)) {
+ values[i] = tp;
+ }
+ }
}
return values;
}
@@ -1977,6 +2008,101 @@ public class TestCubeMetastoreClient {
}
@Test(priority = 2)
+ public void testCubeDimWithNonTimeParts() throws Exception {
+ String dimName = "countrytable_partitioned";
+
+ List<FieldSchema> dimColumns = new ArrayList<FieldSchema>();
+ dimColumns.add(new FieldSchema("id", "int", "code"));
+ dimColumns.add(new FieldSchema("name", "string", "field1"));
+ dimColumns.add(new FieldSchema("capital", "string", "field2"));
+
+ Set<String> storageNames = new HashSet<String>();
+
+ StorageTableDesc s1 = new StorageTableDesc();
+ s1.setInputFormat(TextInputFormat.class.getCanonicalName());
+ s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+ ArrayList<FieldSchema> partCols = Lists.newArrayList();
+ partCols.add(new FieldSchema("region", "string", "region name"));
+ partCols.add(getDatePartition());
+ s1.setPartCols(partCols);
+ s1.setTimePartCols(Collections.singletonList(getDatePartitionKey()));
+ storageNames.add(c3);
+
+ Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>();
+ storageTables.put(c3, s1);
+
+ CubeDimensionTable cubeDim = new CubeDimensionTable(countryDim.getName(), dimName, dimColumns, 0L, storageNames);
+
+ client.createCubeDimensionTable(countryDim.getName(), dimName, dimColumns, 0L, storageNames, null, storageTables);
+ // test partition
+ String storageTableName = MetastoreUtil.getDimStorageTableName(dimName, c3);
+ Assert.assertFalse(client.dimTableLatestPartitionExists(storageTableName));
+ Map<String, Date> expectedLatestValues = Maps.newHashMap();
+ Map<String, Date> timeParts = new HashMap<String, Date>();
+ Map<String, String> nonTimeParts = new HashMap<String, String>();
+
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), now);
+ nonTimeParts.put("region", "asia");
+ StoragePartitionDesc sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, UpdatePeriod.HOURLY);
+ client.addPartition(sPartSpec, c3);
+ expectedLatestValues.put("asia", now);
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), nowMinus1);
+ nonTimeParts.put("region", "africa");
+ sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, UpdatePeriod.HOURLY);
+ client.addPartition(sPartSpec, c3);
+ expectedLatestValues.put("asia", now);
+ expectedLatestValues.put("africa", nowMinus1);
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), nowPlus1);
+ nonTimeParts.put("region", "africa");
+ sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, UpdatePeriod.HOURLY);
+ client.addPartition(sPartSpec, c3);
+ expectedLatestValues.put("asia", now);
+ expectedLatestValues.put("africa", nowPlus1);
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), nowPlus3);
+ nonTimeParts.put("region", "asia");
+ sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, UpdatePeriod.HOURLY);
+ client.addPartition(sPartSpec, c3);
+ expectedLatestValues.put("asia", nowPlus3);
+ expectedLatestValues.put("africa", nowPlus1);
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ client.dropPartition(dimName, c3, timeParts, nonTimeParts, UpdatePeriod.HOURLY);
+ expectedLatestValues.put("asia", now);
+ expectedLatestValues.put("africa", nowPlus1);
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), now);
+ client.dropPartition(dimName, c3, timeParts, nonTimeParts, UpdatePeriod.HOURLY);
+ expectedLatestValues.remove("asia");
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ nonTimeParts.put("region", "africa");
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), nowMinus1);
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), nowPlus3);
+ nonTimeParts.remove("africa");
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+ }
+
+ private void assertLatestForRegions(String storageTableName, Map<String, Date> expectedLatestValues)
+ throws HiveException, LensException {
+ List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
+ Assert.assertEquals(parts.size(), expectedLatestValues.size());
+ for (Partition part : parts) {
+ Assert.assertEquals(MetastoreUtil.getLatestTimeStampOfDimtable(part, getDatePartitionKey()),
+ TimePartition.of(UpdatePeriod.HOURLY, expectedLatestValues.get(part.getSpec().get("region"))).getDate());
+ }
+ }
+
+
+ @Test(priority = 2)
public void testCubeDimWithThreeTimedParts() throws Exception {
String dimName = "ziptableMetaWithThreeTimedParts";
@@ -2059,9 +2185,9 @@ public class TestCubeMetastoreClient {
client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1);
String c1TableName = MetastoreUtil.getDimStorageTableName(cubeDim.getName(), c1);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 6);
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 8);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, now, nowPlus1));
Map<String, Date> timeParts4 = new HashMap<String, Date>();
@@ -2081,8 +2207,8 @@ public class TestCubeMetastoreClient {
client.addPartitions(Arrays.asList(partSpec4, partSpec5), c1);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 8);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 10);
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, nowPlus1, nowPlus1, nowPlus1));
Map<String, Date> timeParts6 = new HashMap<String, Date>();
timeParts6.put(TestCubeMetastoreClient.getDatePartitionKey(), nowMinus2);
@@ -2093,7 +2219,7 @@ public class TestCubeMetastoreClient {
client.addPartition(partSpec6, c1);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 9);
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 11);
Map<String, Date> timeParts7 = new HashMap<String, Date>();
@@ -2104,34 +2230,35 @@ public class TestCubeMetastoreClient {
UpdatePeriod.HOURLY);
client.addPartition(partSpec7, c1);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 10);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 12);
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, nowPlus1, nowPlus1, nowPlus1));
client.dropPartition(cubeDim.getName(), c1, timeParts5, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 9);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 11);
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, nowPlus1, nowPlus1));
client.dropPartition(cubeDim.getName(), c1, timeParts7, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 8);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 10);
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, nowPlus1, nowPlus1));
client.dropPartition(cubeDim.getName(), c1, timeParts2, nonTimeSpec, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 7);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 9);
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, nowPlus1, now));
client.dropPartition(cubeDim.getName(), c1, timeParts4, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 6);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 8);
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, now, now));
client.dropPartition(cubeDim.getName(), c1, timeParts3, nonTimeSpec, UpdatePeriod.HOURLY);
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 5);
client.dropPartition(cubeDim.getName(), c1, timeParts6, null, UpdatePeriod.HOURLY);
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 4);
client.dropPartition(cubeDim.getName(), c1, timeParts1, null, UpdatePeriod.HOURLY);
-
Assert.assertEquals(client.getAllParts(c1TableName).size(), 0);
assertNoPartitionNamedLatest(c1TableName, partColNames);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 517fe7b..19baa00 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -964,7 +964,7 @@ public class CubeTestSetup {
client.clearHiveTableCache();
table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(),
Storage.getPrefix(c4)));
- Assert.assertEquals(table.getParameters().get(MetastoreUtil.getPartitoinTimelineCachePresenceKey()), "true");
+ Assert.assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
Assert.assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineStorageClassKey(UpdatePeriod.DAILY,
"ttd")),
EndsAndHolesPartitionTimeline.class.getCanonicalName());