You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/04/16 12:12:58 UTC
[1/3] incubator-lens git commit: LENS-499 : Allow multiple 'latest'
partitions for different combination of non time partition values (Rajat
Khandelwal via amareshwari)
Repository: incubator-lens
Updated Branches:
refs/heads/current-release-line b10d15be5 -> 992f815a0
LENS-499 : Allow multiple 'latest' partitions for different combination of non time partition values (Rajat Khandelwal via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/120bf6cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/120bf6cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/120bf6cd
Branch: refs/heads/current-release-line
Commit: 120bf6cd7e13949d80aef5fff8ed5ad9b9dd1128
Parents: b10d15b
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Wed Apr 15 15:17:35 2015 -0500
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Apr 15 15:17:35 2015 -0500
----------------------------------------------------------------------
checkstyle/src/main/resources/checkstyle.xml | 2 +-
.../lens/cube/metadata/CubeMetastoreClient.java | 80 +++++----
.../lens/cube/metadata/MetastoreConstants.java | 2 +
.../lens/cube/metadata/MetastoreUtil.java | 50 +++++-
.../org/apache/lens/cube/metadata/Storage.java | 101 ++++++-----
.../lens/cube/metadata/StorageConstants.java | 35 +++-
.../cube/metadata/TestCubeMetastoreClient.java | 171 ++++++++++++++++---
.../apache/lens/cube/parse/CubeTestSetup.java | 2 +-
8 files changed, 340 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/checkstyle/src/main/resources/checkstyle.xml
----------------------------------------------------------------------
diff --git a/checkstyle/src/main/resources/checkstyle.xml b/checkstyle/src/main/resources/checkstyle.xml
index a663f9c..236f977 100644
--- a/checkstyle/src/main/resources/checkstyle.xml
+++ b/checkstyle/src/main/resources/checkstyle.xml
@@ -60,7 +60,7 @@
<module name="NewlineAtEndOfFile"/>
<module name="FileLength">
- <property name="max" value="2500"/>
+ <property name="max" value="3500"/>
</module>
<module name="FileTabCharacter"/>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 324002f..fbdefab 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -188,7 +188,7 @@ public class CubeMetastoreClient {
if (get(storageTableName) == null) {
Table storageTable = getTable(storageTableName);
if ("true".equalsIgnoreCase(storageTable.getParameters().get(
- MetastoreUtil.getPartitoinTimelineCachePresenceKey()))) {
+ MetastoreUtil.getPartitionTimelineCachePresenceKey()))) {
try {
loadTimelinesFromTableProperties(fact, storage);
} catch (Exception e) {
@@ -700,8 +700,14 @@ public class CubeMetastoreClient {
Storage.getPrefix(storageName.trim())).toLowerCase();
if (getDimensionTable(factOrDimTable) != null) {
// Adding partition in dimension table.
+ Map<Map<String, String>, LatestInfo> latestInfos = Maps.newHashMap();
+ for (Map.Entry<Map<String, String>, List<StoragePartitionDesc>> entry : groupByNonTimePartitions(
+ storagePartitionDescs).entrySet()) {
+ latestInfos.put(entry.getKey(),
+ getDimTableLatestInfo(storageTableName, entry.getKey(), getTimePartSpecs(entry.getValue()), updatePeriod));
+ }
getStorage(storageName).addPartitions(getClient(), factOrDimTable, updatePeriod, storagePartitionDescs,
- getDimTableLatestInfo(storageTableName, getTimePartSpecs(storagePartitionDescs), updatePeriod));
+ latestInfos);
latestLookupCache.add(storageTableName);
} else {
// first update in memory, then add to hive table's partitions. delete is reverse.
@@ -759,7 +765,7 @@ public class CubeMetastoreClient {
entry.getValue().updateTableParams(table);
}
}
- params.put(MetastoreUtil.getPartitoinTimelineCachePresenceKey(), "true");
+ params.put(MetastoreUtil.getPartitionTimelineCachePresenceKey(), "true");
alterHiveTable(storageTableName, table);
}
}
@@ -769,7 +775,8 @@ public class CubeMetastoreClient {
return UpdatePeriod.valueOf(partition.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD));
}
- private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, TreeSet<Date>> partitionTimestamps,
+ private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, String> nonTimeParts,
+ Map<String, TreeSet<Date>> timePartSpecs,
UpdatePeriod updatePeriod) throws HiveException {
Table hiveTable = getHiveTable(storageTableName);
String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS);
@@ -777,13 +784,13 @@ public class CubeMetastoreClient {
LatestInfo latest = new LatestInfo();
String[] timePartCols = StringUtils.split(timePartColsStr, ',');
for (String partCol : timePartCols) {
- if (!partitionTimestamps.containsKey(partCol)) {
+ if (!timePartSpecs.containsKey(partCol)) {
continue;
}
boolean makeLatest = true;
- Partition part = getLatestPart(storageTableName, partCol);
- Date pTimestamp = partitionTimestamps.get(partCol).last();
- Date latestTimestamp = getLatestTimeStampOfDimtable(part, partCol);
+ Partition part = getLatestPart(storageTableName, partCol, nonTimeParts);
+ Date pTimestamp = timePartSpecs.get(partCol).last();
+ Date latestTimestamp = MetastoreUtil.getLatestTimeStampOfDimtable(part, partCol);
if (latestTimestamp != null && pTimestamp.before(latestTimestamp)) {
makeLatest = false;
}
@@ -800,22 +807,20 @@ public class CubeMetastoreClient {
}
}
- private boolean isLatestPartOfDimtable(Partition part) {
- return part.getValues().contains(StorageConstants.LATEST_PARTITION_VALUE);
- }
-
- public Date getLatestTimeStampOfDimtable(Partition part, String partCol) throws HiveException {
- if (part != null) {
- String latestTimeStampStr = part.getParameters().get(MetastoreUtil.getLatestPartTimestampKey(partCol));
- String latestPartUpdatePeriod = part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD);
- UpdatePeriod latestUpdatePeriod = UpdatePeriod.valueOf(latestPartUpdatePeriod.toUpperCase());
- try {
- return latestUpdatePeriod.format().parse(latestTimeStampStr);
- } catch (ParseException e) {
- throw new HiveException(e);
+ private Map<Map<String, String>, List<StoragePartitionDesc>> groupByNonTimePartitions(
+ List<StoragePartitionDesc> storagePartitionDescs) {
+ Map<Map<String, String>, List<StoragePartitionDesc>> result = Maps.newHashMap();
+ for (StoragePartitionDesc storagePartitionDesc : storagePartitionDescs) {
+ if (result.get(storagePartitionDesc.getNonTimePartSpec()) == null) {
+ result.put(storagePartitionDesc.getNonTimePartSpec(), Lists.<StoragePartitionDesc>newArrayList());
}
+ result.get(storagePartitionDesc.getNonTimePartSpec()).add(storagePartitionDesc);
}
- return null;
+ return result;
+ }
+
+ private boolean isLatestPartOfDimtable(Partition part) {
+ return part.getValues().contains(StorageConstants.LATEST_PARTITION_VALUE);
}
private Date getPartDate(Partition part, int timeColIndex) {
@@ -833,10 +838,17 @@ public class CubeMetastoreClient {
return partDate;
}
- private LatestInfo getNextLatestOfDimtable(Table hiveTable, String timeCol, final int timeColIndex)
+ private LatestInfo getNextLatestOfDimtable(Table hiveTable, String timeCol, final int timeColIndex,
+ UpdatePeriod updatePeriod, Map<String, String> nonTimePartSpec)
throws HiveException {
// getClient().getPartitionsByNames(tbl, partNames)
- List<Partition> partitions = getClient().getPartitions(hiveTable);
+ List<Partition> partitions = null;
+ try {
+ partitions = getClient().getPartitionsByFilter(hiveTable, StorageConstants.getPartFilter(nonTimePartSpec));
+ MetastoreUtil.filterPartitionsByNonTimeParts(partitions, nonTimePartSpec, timeCol);
+ } catch (TException e) {
+ throw new HiveException(e);
+ }
// tree set contains partitions with timestamp as value for timeCol, in
// descending order
@@ -867,7 +879,7 @@ public class CubeMetastoreClient {
}
}
Iterator<Partition> it = allPartTimeVals.iterator();
- it.next();
+ it.next(); // Skip itself. We have to find next latest.
LatestInfo latest = null;
if (it.hasNext()) {
Partition nextLatest = it.next();
@@ -920,7 +932,7 @@ public class CubeMetastoreClient {
throw new HiveException("Not a time partition column:" + timeCol);
}
int timeColIndex = partColNames.indexOf(timeCol);
- Partition part = getLatestPart(storageTableName, timeCol);
+ Partition part = getLatestPart(storageTableName, timeCol, nonTimePartSpec);
boolean isLatest = true;
// check if partition being dropped is the latest partition
@@ -933,7 +945,7 @@ public class CubeMetastoreClient {
}
}
if (isLatest) {
- Date latestTimestamp = getLatestTimeStampOfDimtable(part, timeCol);
+ Date latestTimestamp = MetastoreUtil.getLatestTimeStampOfDimtable(part, timeCol);
Date dropTimestamp;
try {
dropTimestamp = updatePeriod.format().parse(updatePeriod.format().format(timePartSpec.get(timeCol)));
@@ -941,7 +953,8 @@ public class CubeMetastoreClient {
throw new HiveException(e);
}
if (latestTimestamp != null && dropTimestamp.equals(latestTimestamp)) {
- LatestInfo latestInfo = getNextLatestOfDimtable(hiveTable, timeCol, timeColIndex);
+ LatestInfo latestInfo =
+ getNextLatestOfDimtable(hiveTable, timeCol, timeColIndex, updatePeriod, nonTimePartSpec);
latestAvailable = (latestInfo != null && latestInfo.part != null);
latest.put(timeCol, latestInfo);
}
@@ -954,14 +967,14 @@ public class CubeMetastoreClient {
throw new HiveException("Not time part columns" + timePartSpec.keySet());
}
}
- getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, latest);
+ getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, latest, nonTimePartSpec);
if (!latestAvailable) {
// dropping latest and could not find latest, removing the entry from latest lookup cache
latestLookupCache.remove(storageTableName);
}
} else {
// dropping fact partition
- getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, null);
+ getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, null, null);
if (partitionTimelineCache.updateForDeletion(cubeTableName, storageName, updatePeriod, timePartSpec)) {
this.alterTablePartitionCache(storageTableName);
}
@@ -1083,8 +1096,13 @@ public class CubeMetastoreClient {
}
Partition getLatestPart(String storageTableName, String latestPartCol) throws HiveException {
+ return getLatestPart(storageTableName, latestPartCol, null);
+ }
+
+ Partition getLatestPart(String storageTableName, String latestPartCol, Map<String, String> nonTimeParts)
+ throws HiveException {
List<Partition> latestParts =
- getPartitionsByFilter(storageTableName, StorageConstants.getLatestPartFilter(latestPartCol));
+ getPartitionsByFilter(storageTableName, StorageConstants.getLatestPartFilter(latestPartCol, nonTimeParts));
if (latestParts != null && !latestParts.isEmpty()) {
return latestParts.get(0);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
index e7f10ac..74a26fe 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
@@ -53,7 +53,9 @@ public final class MetastoreConstants {
public static final String FACT_AGGREGATED_PROPERTY = "cube.fact.is.aggregated";
// dim table constants
+ // TODO: remove this and move to "dimtable."
public static final String DIM_TBL_PFX = "dimtble.";
+ public static final String DIM_TABLE_PFX = "dimtable.";
public static final String DUMP_PERIOD_SFX = ".dumpperiod";
public static final String STORAGE_LIST_SFX = ".storages";
public static final String DIM_NAME_SFX = ".dim.name";
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index bdec4e3..8e2c9ed 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -21,10 +21,13 @@ package org.apache.lens.cube.metadata;
import static org.apache.lens.cube.metadata.MetastoreConstants.*;
+import java.text.ParseException;
import java.util.*;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
public class MetastoreUtil {
private MetastoreUtil() {
@@ -63,7 +66,7 @@ public class MetastoreUtil {
}
public static final String getDimTablePartsKey(String dimtableName) {
- return getDimensionTablePrefix(dimtableName) + PARTCOLS_SFX;
+ return DIM_TABLE_PFX + dimtableName + PARTCOLS_SFX;
}
public static final String getDimTimedDimensionKey(String dimName) {
@@ -401,7 +404,7 @@ public class MetastoreUtil {
}
static <E extends Named> void addNameStrings(Map<String, String> props, String key,
- Collection<E> set, int maxLength) {
+ Collection<E> set, int maxLength) {
List<String> namedStrings = getNamedStrs(set, maxLength);
props.put(key + ".size", String.valueOf(namedStrings.size()));
for (int i = 0; i < namedStrings.size(); i++) {
@@ -480,7 +483,48 @@ public class MetastoreUtil {
return getPartitionInfoKeyPrefix(updatePeriod, partCol) + STORAGE_CLASS;
}
- public static String getPartitoinTimelineCachePresenceKey() {
+ public static String getPartitionTimelineCachePresenceKey() {
return STORAGE_PFX + PARTITION_TIMELINE_CACHE + "present";
}
+
+ public static List<Partition> filterPartitionsByNonTimeParts(List<Partition> partitions,
+ Map<String, String> nonTimePartSpec,
+ String latestPartCol) {
+ ListIterator<Partition> iter = partitions.listIterator();
+ while (iter.hasNext()) {
+ Partition part = iter.next();
+ boolean ignore = false;
+ for (Map.Entry<String, String> entry1 : part.getSpec().entrySet()) {
+ if ((nonTimePartSpec == null || !nonTimePartSpec.containsKey(entry1.getKey()))
+ && !entry1.getKey().equals(latestPartCol)) {
+ try {
+ UpdatePeriod.valueOf(part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD))
+ .format()
+ .parse(entry1.getValue());
+ } catch (ParseException e) {
+ ignore = true;
+ }
+ }
+ }
+
+ if (ignore) {
+ iter.remove();
+ }
+ }
+ return partitions;
+ }
+
+ public static Date getLatestTimeStampOfDimtable(Partition part, String partCol) throws HiveException {
+ if (part != null) {
+ String latestTimeStampStr = part.getParameters().get(MetastoreUtil.getLatestPartTimestampKey(partCol));
+ String latestPartUpdatePeriod = part.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD);
+ UpdatePeriod latestUpdatePeriod = UpdatePeriod.valueOf(latestPartUpdatePeriod.toUpperCase());
+ try {
+ return latestUpdatePeriod.format().parse(latestTimeStampStr);
+ } catch (ParseException e) {
+ throw new HiveException(e);
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
index 50ec4ce..d3c9974 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
@@ -237,25 +237,28 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
*/
public void addPartition(Hive client, StoragePartitionDesc addPartitionDesc, LatestInfo latestInfo)
throws HiveException {
- addPartitions(client, addPartitionDesc.getCubeTableName(), addPartitionDesc.getUpdatePeriod(), Arrays.asList(
- addPartitionDesc), latestInfo);
+ Map<Map<String, String>, LatestInfo> latestInfos = Maps.newHashMap();
+ latestInfos.put(addPartitionDesc.getNonTimePartSpec(), latestInfo);
+ addPartitions(client, addPartitionDesc.getCubeTableName(), addPartitionDesc.getUpdatePeriod(),
+ Collections.singletonList(addPartitionDesc), latestInfos);
}
/**
* Add given partitions in the underlying hive table and update latest partition links
*
* @param client hive client instance
- * @param fact fact name
+ * @param factOrDimTable fact or dim name
* @param updatePeriod update period of partitions.
* @param storagePartitionDescs all partitions to be added
- * @param latestInfo new latest info. atleast one partition for the latest value exists for each part
+ * @param latestInfos new latest info. atleast one partition for the latest value exists for each part
* column
* @throws HiveException
*/
public void addPartitions(Hive client, String factOrDimTable, UpdatePeriod updatePeriod,
- List<StoragePartitionDesc> storagePartitionDescs, LatestInfo latestInfo) throws HiveException {
+ List<StoragePartitionDesc> storagePartitionDescs,
+ Map<Map<String, String>, LatestInfo> latestInfos) throws HiveException {
preAddPartitions(storagePartitionDescs);
- Map<String, Integer> latestPartIndexForPartCols = Maps.newHashMap();
+ Map<Map<String, String>, Map<String, Integer>> latestPartIndexForPartCols = Maps.newHashMap();
boolean success = false;
try {
String tableName = MetastoreUtil.getStorageTableName(factOrDimTable, this.getPrefix());
@@ -288,46 +291,56 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
addParts.getPartition(curIndex).setSerdeParams(addPartitionDesc.getSerdeParams());
addParts.getPartition(curIndex).setBucketCols(addPartitionDesc.getBucketCols());
addParts.getPartition(curIndex).setSortCols(addPartitionDesc.getSortCols());
- if (latestInfo != null) {
- for (Map.Entry<String, LatestPartColumnInfo> entry : latestInfo.latestParts.entrySet()) {
+ if (latestInfos != null && latestInfos.get(addPartitionDesc.getNonTimePartSpec()) != null) {
+ for (Map.Entry<String, LatestPartColumnInfo> entry : latestInfos
+ .get(addPartitionDesc.getNonTimePartSpec()).latestParts.entrySet()) {
if (addPartitionDesc.getTimePartSpec().containsKey(entry.getKey())
&& entry.getValue().get(MetastoreUtil.getLatestPartTimestampKey(entry.getKey())).equals(
updatePeriod.format().format(addPartitionDesc.getTimePartSpec().get(entry.getKey())))) {
- latestPartIndexForPartCols.put(entry.getKey(), curIndex);
+ if (latestPartIndexForPartCols.get(addPartitionDesc.getNonTimePartSpec()) == null) {
+ latestPartIndexForPartCols.put(addPartitionDesc.getNonTimePartSpec(),
+ Maps.<String, Integer>newHashMap());
+ }
+ latestPartIndexForPartCols.get(addPartitionDesc.getNonTimePartSpec()).put(entry.getKey(), curIndex);
}
}
}
}
- if (latestInfo != null) {
- for (Map.Entry<String, LatestPartColumnInfo> entry : latestInfo.latestParts.entrySet()) {
- // symlink this partition to latest
- List<Partition> latest;
- String latestPartCol = entry.getKey();
- try {
- latest = client.getPartitionsByFilter(storageTbl, StorageConstants.getLatestPartFilter(latestPartCol));
- } catch (Exception e) {
- throw new HiveException("Could not get latest partition", e);
- }
- if (!latest.isEmpty()) {
- client.dropPartition(storageTbl.getTableName(), latest.get(0).getValues(), false);
- }
- if (latestPartIndexForPartCols.containsKey(latestPartCol)) {
- AddPartitionDesc.OnePartitionDesc latestPartWithFullTimestamp = addParts.getPartition(
- latestPartIndexForPartCols.get(latestPartCol));
- addParts.addPartition(
- StorageConstants.getLatestPartSpec(latestPartWithFullTimestamp.getPartSpec(), latestPartCol),
- latestPartWithFullTimestamp.getLocation());
- int curIndex = addParts.getPartitionCount() - 1;
- addParts.getPartition(curIndex).setPartParams(entry.getValue().getPartParams(
- latestPartWithFullTimestamp.getPartParams()));
- addParts.getPartition(curIndex).setInputFormat(latestPartWithFullTimestamp.getInputFormat());
- addParts.getPartition(curIndex).setOutputFormat(latestPartWithFullTimestamp.getOutputFormat());
- addParts.getPartition(curIndex).setNumBuckets(latestPartWithFullTimestamp.getNumBuckets());
- addParts.getPartition(curIndex).setCols(latestPartWithFullTimestamp.getCols());
- addParts.getPartition(curIndex).setSerializationLib(latestPartWithFullTimestamp.getSerializationLib());
- addParts.getPartition(curIndex).setSerdeParams(latestPartWithFullTimestamp.getSerdeParams());
- addParts.getPartition(curIndex).setBucketCols(latestPartWithFullTimestamp.getBucketCols());
- addParts.getPartition(curIndex).setSortCols(latestPartWithFullTimestamp.getSortCols());
+ if (latestInfos != null) {
+ for (Map.Entry<Map<String, String>, LatestInfo> entry1 : latestInfos.entrySet()) {
+ Map<String, String> nonTimeParts = entry1.getKey();
+ LatestInfo latestInfo = entry1.getValue();
+ for (Map.Entry<String, LatestPartColumnInfo> entry : latestInfo.latestParts.entrySet()) {
+ // symlink this partition to latest
+ List<Partition> latest;
+ String latestPartCol = entry.getKey();
+ try {
+ latest = client
+ .getPartitionsByFilter(storageTbl, StorageConstants.getLatestPartFilter(latestPartCol, nonTimeParts));
+ } catch (Exception e) {
+ throw new HiveException("Could not get latest partition", e);
+ }
+ if (!latest.isEmpty()) {
+ client.dropPartition(storageTbl.getTableName(), latest.get(0).getValues(), false);
+ }
+ if (latestPartIndexForPartCols.get(nonTimeParts).containsKey(latestPartCol)) {
+ AddPartitionDesc.OnePartitionDesc latestPartWithFullTimestamp = addParts.getPartition(
+ latestPartIndexForPartCols.get(nonTimeParts).get(latestPartCol));
+ addParts.addPartition(
+ StorageConstants.getLatestPartSpec(latestPartWithFullTimestamp.getPartSpec(), latestPartCol),
+ latestPartWithFullTimestamp.getLocation());
+ int curIndex = addParts.getPartitionCount() - 1;
+ addParts.getPartition(curIndex).setPartParams(entry.getValue().getPartParams(
+ latestPartWithFullTimestamp.getPartParams()));
+ addParts.getPartition(curIndex).setInputFormat(latestPartWithFullTimestamp.getInputFormat());
+ addParts.getPartition(curIndex).setOutputFormat(latestPartWithFullTimestamp.getOutputFormat());
+ addParts.getPartition(curIndex).setNumBuckets(latestPartWithFullTimestamp.getNumBuckets());
+ addParts.getPartition(curIndex).setCols(latestPartWithFullTimestamp.getCols());
+ addParts.getPartition(curIndex).setSerializationLib(latestPartWithFullTimestamp.getSerializationLib());
+ addParts.getPartition(curIndex).setSerdeParams(latestPartWithFullTimestamp.getSerdeParams());
+ addParts.getPartition(curIndex).setBucketCols(latestPartWithFullTimestamp.getBucketCols());
+ addParts.getPartition(curIndex).setSortCols(latestPartWithFullTimestamp.getSortCols());
+ }
}
}
}
@@ -348,10 +361,11 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
* @param storageTableName TableName
* @param partVals Partition specification
* @param updateLatestInfo The latest partition info if it needs update, null if latest should not be updated
+ * @param nonTimePartSpec
* @throws HiveException
*/
public void dropPartition(Hive client, String storageTableName, List<String> partVals,
- Map<String, LatestInfo> updateLatestInfo) throws HiveException {
+ Map<String, LatestInfo> updateLatestInfo, Map<String, String> nonTimePartSpec) throws HiveException {
preDropPartition(storageTableName, partVals);
boolean success = false;
try {
@@ -360,16 +374,19 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
Table storageTbl = client.getTable(storageTableName);
// update latest info
if (updateLatestInfo != null) {
- for (Map.Entry<String, LatestInfo> entry : updateLatestInfo.entrySet()) {
+ for (Entry<String, LatestInfo> entry : updateLatestInfo.entrySet()) {
String latestPartCol = entry.getKey();
// symlink this partition to latest
List<Partition> latestParts;
try {
- latestParts = client.getPartitionsByFilter(storageTbl, StorageConstants.getLatestPartFilter(latestPartCol));
+ latestParts = client.getPartitionsByFilter(storageTbl,
+ StorageConstants.getLatestPartFilter(latestPartCol, nonTimePartSpec));
+ MetastoreUtil.filterPartitionsByNonTimeParts(latestParts, nonTimePartSpec, latestPartCol);
} catch (Exception e) {
throw new HiveException("Could not get latest partition", e);
}
if (!latestParts.isEmpty()) {
+ assert latestParts.size() == 1;
client.dropPartition(storageTbl.getTableName(), latestParts.get(0).getValues(), false);
}
LatestInfo latest = entry.getValue();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
index 610d168..5881266 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
@@ -24,6 +24,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import com.google.common.collect.Maps;
+
public final class StorageConstants {
private StorageConstants() {
@@ -40,11 +42,38 @@ public final class StorageConstants {
* @return latest partition spec as Map from String to String
*/
public static String getLatestPartFilter(String partCol) {
- return partCol + "='" + LATEST_PARTITION_VALUE + "'";
+ return getPartFilter(partCol, LATEST_PARTITION_VALUE);
+ }
+
+ public static String getPartFilter(final String partCol, final String value) {
+ return getPartFilter(new HashMap<String, String>() {
+ {
+ put(partCol, value);
+ }
+ });
+ }
+
+ public static String getPartFilter(Map<String, String> parts) {
+ String sep = "";
+ StringBuilder ret = new StringBuilder();
+ if (parts != null) {
+ for (Map.Entry<String, String> entry : parts.entrySet()) {
+ ret.append(sep).append(entry.getKey()).append("='").append(entry.getValue()).append("'");
+ sep = " and ";
+ }
+ }
+ return ret.toString();
+ }
+
+ public static String getPartFilter(String partCol, String value, Map<String, String> parts) {
+ Map<String, String> allParts = Maps.newHashMap();
+ allParts.putAll(parts);
+ allParts.put(partCol, value);
+ return getPartFilter(allParts);
}
- public static String getPartFilter(String partCol, String value) {
- return partCol + "='" + value + "'";
+ public static String getLatestPartFilter(String partCol, Map<String, String> parts) {
+ return getPartFilter(partCol, LATEST_PARTITION_VALUE, parts);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 96f2b9b..d4c109a 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -47,6 +47,9 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
public class TestCubeMetastoreClient {
private static CubeMetastoreClient client;
@@ -67,6 +70,8 @@ public class TestCubeMetastoreClient {
private static final Map<String, String> CUBE_PROPERTIES = new HashMap<String, String>();
private static Date now;
private static Date nowPlus1;
+ private static Date nowPlus2;
+ private static Date nowPlus3;
private static Date nowMinus1;
private static Date nowMinus2;
private static Date nowMinus3;
@@ -114,6 +119,10 @@ public class TestCubeMetastoreClient {
cal.setTime(now);
cal.add(Calendar.HOUR_OF_DAY, 1);
nowPlus1 = cal.getTime();
+ cal.add(Calendar.HOUR_OF_DAY, 1);
+ nowPlus2 = cal.getTime();
+ cal.add(Calendar.HOUR_OF_DAY, 1);
+ nowPlus3 = cal.getTime();
cal.setTime(now);
cal.add(Calendar.HOUR, -1);
nowMinus1 = cal.getTime();
@@ -945,7 +954,7 @@ public class TestCubeMetastoreClient {
String c4TableName = MetastoreUtil.getFactStorageTableName(factName, c4);
Assert.assertTrue(client.tableExists(c4TableName));
- // add storage
+ // add storage
client.addStorage(altered2, c3, updates, s1);
CubeFactTable altered3 = client.getCubeFact(factName);
Assert.assertTrue(altered3.getStorages().contains("C3"));
@@ -1411,14 +1420,36 @@ public class TestCubeMetastoreClient {
}
}
- private TimePartition[] getLatestValues(String storageTableName, UpdatePeriod updatePeriod, String... partCols)
+ private TimePartition[] getLatestValues(String storageTableName, UpdatePeriod updatePeriod, String[] partCols,
+ Map<String, String> nonTimeParts)
throws LensException, HiveException {
TimePartition[] values = new TimePartition[partCols.length];
for (int i = 0; i < partCols.length; i++) {
- List<Partition> part = client.getPartitionsByFilter(storageTableName, partCols[i] + "='latest'");
- Assert.assertEquals(part.size(), 1);
- values[i] = TimePartition.of(updatePeriod, part.get(0).getParameters().get(
- MetastoreUtil.getLatestPartTimestampKey(partCols[i])));
+ List<Partition> parts = client.getPartitionsByFilter(storageTableName,
+ StorageConstants.getLatestPartFilter(partCols[i], nonTimeParts));
+ for (Partition part : parts) {
+ boolean ignore = false;
+ if (nonTimeParts != null) {
+ for (Map.Entry<String, String> entry : part.getSpec().entrySet()) {
+ if (!nonTimeParts.containsKey(entry.getKey())) {
+ try {
+ updatePeriod.format().parse(entry.getValue());
+ } catch (java.text.ParseException e) {
+ ignore = true;
+ break;
+ }
+ }
+ }
+ }
+ if (ignore) {
+ break;
+ }
+ TimePartition tp = TimePartition.of(updatePeriod, part.getParameters().get(
+ MetastoreUtil.getLatestPartTimestampKey(partCols[i])));
+ if (values[i] == null || values[i].before(tp)) {
+ values[i] = tp;
+ }
+ }
}
return values;
}
@@ -1977,6 +2008,101 @@ public class TestCubeMetastoreClient {
}
@Test(priority = 2)
+ public void testCubeDimWithNonTimeParts() throws Exception {
+ String dimName = "countrytable_partitioned";
+
+ List<FieldSchema> dimColumns = new ArrayList<FieldSchema>();
+ dimColumns.add(new FieldSchema("id", "int", "code"));
+ dimColumns.add(new FieldSchema("name", "string", "field1"));
+ dimColumns.add(new FieldSchema("capital", "string", "field2"));
+
+ Set<String> storageNames = new HashSet<String>();
+
+ StorageTableDesc s1 = new StorageTableDesc();
+ s1.setInputFormat(TextInputFormat.class.getCanonicalName());
+ s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+ ArrayList<FieldSchema> partCols = Lists.newArrayList();
+ partCols.add(new FieldSchema("region", "string", "region name"));
+ partCols.add(getDatePartition());
+ s1.setPartCols(partCols);
+ s1.setTimePartCols(Collections.singletonList(getDatePartitionKey()));
+ storageNames.add(c3);
+
+ Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>();
+ storageTables.put(c3, s1);
+
+ CubeDimensionTable cubeDim = new CubeDimensionTable(countryDim.getName(), dimName, dimColumns, 0L, storageNames);
+
+ client.createCubeDimensionTable(countryDim.getName(), dimName, dimColumns, 0L, storageNames, null, storageTables);
+ // test partition
+ String storageTableName = MetastoreUtil.getDimStorageTableName(dimName, c3);
+ Assert.assertFalse(client.dimTableLatestPartitionExists(storageTableName));
+ Map<String, Date> expectedLatestValues = Maps.newHashMap();
+ Map<String, Date> timeParts = new HashMap<String, Date>();
+ Map<String, String> nonTimeParts = new HashMap<String, String>();
+
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), now);
+ nonTimeParts.put("region", "asia");
+ StoragePartitionDesc sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, UpdatePeriod.HOURLY);
+ client.addPartition(sPartSpec, c3);
+ expectedLatestValues.put("asia", now);
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), nowMinus1);
+ nonTimeParts.put("region", "africa");
+ sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, UpdatePeriod.HOURLY);
+ client.addPartition(sPartSpec, c3);
+ expectedLatestValues.put("asia", now);
+ expectedLatestValues.put("africa", nowMinus1);
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), nowPlus1);
+ nonTimeParts.put("region", "africa");
+ sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, UpdatePeriod.HOURLY);
+ client.addPartition(sPartSpec, c3);
+ expectedLatestValues.put("asia", now);
+ expectedLatestValues.put("africa", nowPlus1);
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), nowPlus3);
+ nonTimeParts.put("region", "asia");
+ sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, UpdatePeriod.HOURLY);
+ client.addPartition(sPartSpec, c3);
+ expectedLatestValues.put("asia", nowPlus3);
+ expectedLatestValues.put("africa", nowPlus1);
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ client.dropPartition(dimName, c3, timeParts, nonTimeParts, UpdatePeriod.HOURLY);
+ expectedLatestValues.put("asia", now);
+ expectedLatestValues.put("africa", nowPlus1);
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), now);
+ client.dropPartition(dimName, c3, timeParts, nonTimeParts, UpdatePeriod.HOURLY);
+ expectedLatestValues.remove("asia");
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ nonTimeParts.put("region", "africa");
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), nowMinus1);
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+
+ timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), nowPlus3);
+ nonTimeParts.remove("africa");
+ assertLatestForRegions(storageTableName, expectedLatestValues);
+ }
+
+ private void assertLatestForRegions(String storageTableName, Map<String, Date> expectedLatestValues)
+ throws HiveException, LensException {
+ List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
+ Assert.assertEquals(parts.size(), expectedLatestValues.size());
+ for (Partition part : parts) {
+ Assert.assertEquals(MetastoreUtil.getLatestTimeStampOfDimtable(part, getDatePartitionKey()),
+ TimePartition.of(UpdatePeriod.HOURLY, expectedLatestValues.get(part.getSpec().get("region"))).getDate());
+ }
+ }
+
+
+ @Test(priority = 2)
public void testCubeDimWithThreeTimedParts() throws Exception {
String dimName = "ziptableMetaWithThreeTimedParts";
@@ -2059,9 +2185,9 @@ public class TestCubeMetastoreClient {
client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1);
String c1TableName = MetastoreUtil.getDimStorageTableName(cubeDim.getName(), c1);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 6);
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 8);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, now, nowPlus1));
Map<String, Date> timeParts4 = new HashMap<String, Date>();
@@ -2081,8 +2207,8 @@ public class TestCubeMetastoreClient {
client.addPartitions(Arrays.asList(partSpec4, partSpec5), c1);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 8);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 10);
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, nowPlus1, nowPlus1, nowPlus1));
Map<String, Date> timeParts6 = new HashMap<String, Date>();
timeParts6.put(TestCubeMetastoreClient.getDatePartitionKey(), nowMinus2);
@@ -2093,7 +2219,7 @@ public class TestCubeMetastoreClient {
client.addPartition(partSpec6, c1);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 9);
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 11);
Map<String, Date> timeParts7 = new HashMap<String, Date>();
@@ -2104,34 +2230,35 @@ public class TestCubeMetastoreClient {
UpdatePeriod.HOURLY);
client.addPartition(partSpec7, c1);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 10);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 12);
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, nowPlus1, nowPlus1, nowPlus1));
client.dropPartition(cubeDim.getName(), c1, timeParts5, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 9);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 11);
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, nowPlus1, nowPlus1));
client.dropPartition(cubeDim.getName(), c1, timeParts7, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 8);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 10);
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, nowPlus1, nowPlus1));
client.dropPartition(cubeDim.getName(), c1, timeParts2, nonTimeSpec, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 7);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 9);
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, nowPlus1, now));
client.dropPartition(cubeDim.getName(), c1, timeParts4, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 6);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames),
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 8);
+ Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, now, now));
client.dropPartition(cubeDim.getName(), c1, timeParts3, nonTimeSpec, UpdatePeriod.HOURLY);
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 5);
client.dropPartition(cubeDim.getName(), c1, timeParts6, null, UpdatePeriod.HOURLY);
+ Assert.assertEquals(client.getAllParts(c1TableName).size(), 4);
client.dropPartition(cubeDim.getName(), c1, timeParts1, null, UpdatePeriod.HOURLY);
-
Assert.assertEquals(client.getAllParts(c1TableName).size(), 0);
assertNoPartitionNamedLatest(c1TableName, partColNames);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/120bf6cd/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 517fe7b..19baa00 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -964,7 +964,7 @@ public class CubeTestSetup {
client.clearHiveTableCache();
table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(),
Storage.getPrefix(c4)));
- Assert.assertEquals(table.getParameters().get(MetastoreUtil.getPartitoinTimelineCachePresenceKey()), "true");
+ Assert.assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
Assert.assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineStorageClassKey(UpdatePeriod.DAILY,
"ttd")),
EndsAndHolesPartitionTimeline.class.getCanonicalName());
[3/3] incubator-lens git commit: LENS-499: (incremental) Handle NPE
in test cases
Posted by am...@apache.org.
LENS-499: (incremental) Handle NPE in test cases
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/992f815a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/992f815a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/992f815a
Branch: refs/heads/current-release-line
Commit: 992f815a09718a8449f15daa23f52304c1f12961
Parents: 001970d
Author: Rajat Khandelwal <ra...@gmail.com>
Authored: Thu Apr 16 12:52:19 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Apr 16 12:52:19 2015 +0530
----------------------------------------------------------------------
.../java/org/apache/lens/cube/metadata/StorageConstants.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/992f815a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
index 5881266..8c1bca7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java
@@ -67,7 +67,9 @@ public final class StorageConstants {
public static String getPartFilter(String partCol, String value, Map<String, String> parts) {
Map<String, String> allParts = Maps.newHashMap();
- allParts.putAll(parts);
+ if (parts != null) {
+ allParts.putAll(parts);
+ }
allParts.put(partCol, value);
return getPartFilter(allParts);
}
[2/3] incubator-lens git commit: LENS-436 : Fix dependencies in ML
distribution
Posted by am...@apache.org.
LENS-436 : Fix dependencies in ML distribution
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/001970de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/001970de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/001970de
Branch: refs/heads/current-release-line
Commit: 001970de5ff73bb6effba630c474550b286b2a3b
Parents: 120bf6c
Author: Sharad Agarwal <sh...@apache.org>
Authored: Thu Apr 16 10:58:18 2015 +0530
Committer: jdhok <jd...@apache.org>
Committed: Thu Apr 16 10:58:18 2015 +0530
----------------------------------------------------------------------
lens-dist/src/main/assembly/bin-dist.xml | 1 +
lens-docker/lens-test/Dockerfile | 7 ++++
lens-docker/lens-test/lens-bootstrap.sh | 10 ++++-
lens-ml-dist/pom.xml | 40 ++++++++++++++++++++
lens-ml-dist/src/main/assembly/ml-dist.xml | 2 +-
.../org/apache/lens/ml/impl/LensMLImpl.java | 14 +------
.../apache/lens/ml/server/MLServiceImpl.java | 4 --
lens-ml-lib/src/test/resources/lens-site.xml | 4 --
pom.xml | 2 +-
.../server/hivedriver-site.xml | 4 ++
10 files changed, 63 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-dist/src/main/assembly/bin-dist.xml
----------------------------------------------------------------------
diff --git a/lens-dist/src/main/assembly/bin-dist.xml b/lens-dist/src/main/assembly/bin-dist.xml
index b7b74d5..9e2ce8f 100644
--- a/lens-dist/src/main/assembly/bin-dist.xml
+++ b/lens-dist/src/main/assembly/bin-dist.xml
@@ -118,6 +118,7 @@
<include>lens-config.sh</include>
<include>lens-cli.sh</include>
<include>lens-cli</include>
+ <include>lens-run-class.sh</include>
</includes>
</fileSet>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-docker/lens-test/Dockerfile
----------------------------------------------------------------------
diff --git a/lens-docker/lens-test/Dockerfile b/lens-docker/lens-test/Dockerfile
index c4886c2..a405b0f 100644
--- a/lens-docker/lens-test/Dockerfile
+++ b/lens-docker/lens-test/Dockerfile
@@ -17,6 +17,13 @@
FROM inmobi/docker-hive
+RUN wget http://apache.mirrors.lucidnetworks.net/spark/spark-1.3.0/spark-1.3.0-bin-hadoop2.4.tgz
+RUN gunzip spark-1.3.0-bin-hadoop2.4.tgz
+RUN tar -xvf spark-1.3.0-bin-hadoop2.4.tar
+RUN mv spark-1.3.0-bin-hadoop2.4 /usr/local
+ENV SPARK_HOME /usr/local/spark-1.3.0-bin-hadoop2.4
+RUN rm spark-1.3.0-bin-hadoop2.4.tar
+
ENV LENS_VERSION 2.1.0-beta-incubating-SNAPSHOT
ENV BASEDIR /opt/lens
ENV LENS_HOME $BASEDIR/lens-dist/target/apache-lens-${LENS_VERSION}-bin/apache-lens-${LENS_VERSION}-bin/server
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-docker/lens-test/lens-bootstrap.sh
----------------------------------------------------------------------
diff --git a/lens-docker/lens-test/lens-bootstrap.sh b/lens-docker/lens-test/lens-bootstrap.sh
index 19685bb..d4935be 100644
--- a/lens-docker/lens-test/lens-bootstrap.sh
+++ b/lens-docker/lens-test/lens-bootstrap.sh
@@ -22,11 +22,17 @@ echo "LENS_SERVER_CONF " $LENS_SERVER_CONF
echo "LENS_CLIENT " $LENS_CLIENT
echo "LENS_CLIENT_CONF " $LENS_CLIENT_CONF
echo "LENS_ML " $LENS_ML
+echo "SPARK_HOME " $SPARK_HOME
#set ml classpath into LENS_EXT_CLASSPATH
LENS_EXT_CLASSPATH=$LENS_EXT_CLASSPATH:`$LENS_ML/bin/lens-ml-classpath.sh`
export LENS_EXT_CLASSPATH
+HIVE_AUX_JARS_PATH=$LENS_ML/lib/lens-ml-lib-2.1.0-beta-incubating-SNAPSHOT.jar,$SPARK_HOME/lib/spark-assembly-1.3.0-hadoop2.4.0.jar
+export HIVE_AUX_JARS_PATH
+
+echo "HIVE_AUX_JARS_PATH " $HIVE_AUX_JARS_PATH
+
#start hive bootstrap script
/etc/hive-bootstrap.sh
@@ -37,8 +43,8 @@ sleep 20
echo "Starting Lens server..."
$LENS_HOME/bin/lens-ctl start --conf $LENS_SERVER_CONF
-echo "Waiting for 40 secs for Lens Server to start ..."
-sleep 40
+echo "Waiting for 60 secs for Lens Server to start ..."
+sleep 60
#Setting up client
$LENS_CLIENT/bin/run-examples.sh sample-metastore --conf $LENS_CLIENT_CONF
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-ml-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-dist/pom.xml b/lens-ml-dist/pom.xml
index 9e4fd1d..7b6f760 100644
--- a/lens-ml-dist/pom.xml
+++ b/lens-ml-dist/pom.xml
@@ -37,6 +37,46 @@
<groupId>org.apache.lens</groupId>
<artifactId>lens-ml-lib</artifactId>
<version>${project.version}</version>
+ <scope>runtime</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>lens-server-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>lens-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>lens-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-multipart</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.test-framework.providers</groupId>
+ <artifactId>jersey-test-framework-provider-grizzly2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.test-framework</groupId>
+ <artifactId>jersey-test-framework-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ </exclusions>
+
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-ml-dist/src/main/assembly/ml-dist.xml
----------------------------------------------------------------------
diff --git a/lens-ml-dist/src/main/assembly/ml-dist.xml b/lens-ml-dist/src/main/assembly/ml-dist.xml
index 30b6f23..50c7375 100644
--- a/lens-ml-dist/src/main/assembly/ml-dist.xml
+++ b/lens-ml-dist/src/main/assembly/ml-dist.xml
@@ -36,7 +36,7 @@
<scope>runtime</scope>
<outputDirectory>lib</outputDirectory>
<includes>
- <include>org.apache.lens:lens-ml</include>
+ <include>org.apache.lens:lens-ml-lib</include>
</includes>
<useTransitiveFiltering>true</useTransitiveFiltering>
<useTransitiveDependencies>true</useTransitiveDependencies>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
index 39617df..aff7c84 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
@@ -695,24 +695,12 @@ public class LensMLImpl implements LensML {
}
LOG.info("Registering UDF for session " + sessionHandle.getPublicId().toString());
- // We have to add UDF jars to the session
- try {
- SessionService sessionService = (SessionService) MLUtils.getServiceProvider().getService(SessionService.NAME);
- String[] udfJars = conf.getStrings("lens.server.ml.predict.udf.jars");
- if (udfJars != null) {
- for (String jar : udfJars) {
- sessionService.addResource(sessionHandle, "jar", jar);
- LOG.info(jar + " added UDF session " + sessionHandle.getPublicId().toString());
- }
- }
- } catch (Exception e) {
- throw new LensException(e);
- }
String regUdfQuery = "CREATE TEMPORARY FUNCTION " + HiveMLUDF.UDF_NAME + " AS '" + HiveMLUDF.class
.getCanonicalName() + "'";
queryRunner.setQueryName("register_predict_udf_" + sessionHandle.getPublicId().toString());
QueryHandle udfQuery = queryRunner.runQuery(regUdfQuery);
+ LOG.info("udf query handle is " + udfQuery);
predictUdfStatus.put(sessionHandle, true);
LOG.info("Predict UDF registered for session " + sessionHandle.getPublicId().toString());
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
index dc29aef..264ca51 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
@@ -29,7 +29,6 @@ import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.ml.algo.api.*;
import org.apache.lens.ml.api.MLTestReport;
-import org.apache.lens.ml.impl.HiveMLUDF;
import org.apache.lens.ml.impl.LensMLImpl;
import org.apache.lens.ml.impl.ModelLoader;
import org.apache.lens.ml.impl.QueryRunner;
@@ -41,7 +40,6 @@ import org.apache.lens.server.api.query.QueryExecutionService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hive.service.CompositeService;
/**
@@ -284,8 +282,6 @@ public class MLServiceImpl extends CompositeService implements MLService {
*/
@Override
public QueryHandle runQuery(String testQuery) throws LensException {
- FunctionRegistry.registerTemporaryFunction("predict", HiveMLUDF.class);
- LOG.info("Registered predict UDF");
// Run the query in query executions service
QueryExecutionService queryService = (QueryExecutionService) getServiceProvider().getService("query");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/lens-ml-lib/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/resources/lens-site.xml b/lens-ml-lib/src/test/resources/lens-site.xml
index 2e1ddab..3d5dbef 100644
--- a/lens-ml-lib/src/test/resources/lens-site.xml
+++ b/lens-ml-lib/src/test/resources/lens-site.xml
@@ -159,8 +159,4 @@
<description>Implementation class for session service</description>
</property>
- <property>
- <name>lens.server.ml.predict.udf.jars</name>
- <value>testjars/serde.jar,testjars/test.jar</value>
- </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d05e55a..66f7824 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,7 @@
<!-- hadoop stack -->
<hadoop.version>2.4.0</hadoop.version>
- <spark.version>1.1.0</spark.version>
+ <spark.version>1.3.0</spark.version>
<hive.version>0.13.3-inm</hive.version>
<!-- common library/framework versions -->
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/001970de/tools/conf-pseudo-distr/server/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/tools/conf-pseudo-distr/server/hivedriver-site.xml b/tools/conf-pseudo-distr/server/hivedriver-site.xml
index 0d14777..ec5954c 100644
--- a/tools/conf-pseudo-distr/server/hivedriver-site.xml
+++ b/tools/conf-pseudo-distr/server/hivedriver-site.xml
@@ -50,4 +50,8 @@
<value>local,cluster</value>
<final>true</final>
</property>
+<property>
+ <name>hive.aux.jars.path</name>
+ <value>file:///opt/lens/lens-ml-dist/target/apache-lens-2.1.0-beta-incubating-SNAPSHOT-ml/lib/lens-ml-lib-2.1.0-beta-incubating-SNAPSHOT.jar,file:///usr/local/spark-1.3.0-bin-hadoop2.4/lib/spark-assembly-1.3.0-hadoop2.4.0.jar</value>
+</property>
</configuration>