You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/10/02 04:37:52 UTC
[17/22] hive git commit: HIVE-11898 : support default partition in
metastoredirectsql (Sergey Shelukhin, reviewed by Sushanth Sowmyan)
HIVE-11898 : support default partition in metastoredirectsql (Sergey Shelukhin, reviewed by Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/116c3e3b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/116c3e3b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/116c3e3b
Branch: refs/heads/llap
Commit: 116c3e3b2d2b2b22fef9abed8f092b04155784cd
Parents: a953b43
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Oct 1 14:11:58 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Oct 1 14:11:58 2015 -0700
----------------------------------------------------------------------
.../hive/metastore/MetaStoreDirectSql.java | 34 +++++++++++++-------
1 file changed, 22 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/116c3e3b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 1f89b7c..95b1ccc 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -109,6 +109,7 @@ class MetaStoreDirectSql {
private final DB dbType;
private final int batchSize;
private final boolean convertMapNullsToEmptyStrings;
+ private final String defaultPartName;
/**
* Whether direct SQL can be used with the current datastore backing {@link #pm}.
@@ -116,6 +117,7 @@ class MetaStoreDirectSql {
private final boolean isCompatibleDatastore;
private final boolean isAggregateStatsCacheEnabled;
private AggregateStatsCache aggrStatsCache;
+
public MetaStoreDirectSql(PersistenceManager pm, Configuration conf) {
this.pm = pm;
this.dbType = determineDbType();
@@ -127,6 +129,7 @@ class MetaStoreDirectSql {
convertMapNullsToEmptyStrings =
HiveConf.getBoolVar(conf, ConfVars.METASTORE_ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS);
+ defaultPartName = HiveConf.getVar(conf, ConfVars.DEFAULTPARTITIONNAME);
String jdoIdFactory = HiveConf.getVar(conf, ConfVars.METASTORE_IDENTIFIER_FACTORY);
if (! ("datanucleus1".equalsIgnoreCase(jdoIdFactory))){
@@ -390,7 +393,7 @@ class MetaStoreDirectSql {
// Derby and Oracle do not interpret filters ANSI-properly in some cases and need a workaround.
boolean dbHasJoinCastBug = (dbType == DB.DERBY || dbType == DB.ORACLE);
String sqlFilter = PartitionFilterGenerator.generateSqlFilter(
- table, tree, params, joins, dbHasJoinCastBug);
+ table, tree, params, joins, dbHasJoinCastBug, defaultPartName);
if (sqlFilter == null) {
return null; // Cannot make SQL filter to push down.
}
@@ -490,8 +493,8 @@ class MetaStoreDirectSql {
}
List<Object> sqlResult = executeWithArray(query, params, queryText);
long queryTime = doTrace ? System.nanoTime() : 0;
+ timingTrace(doTrace, queryText, start, queryTime);
if (sqlResult.isEmpty()) {
- timingTrace(doTrace, queryText, start, queryTime);
return new ArrayList<Partition>(); // no partitions, bail early.
}
@@ -508,7 +511,6 @@ class MetaStoreDirectSql {
result = getPartitionsFromPartitionIds(dbName, tblName, isView, sqlResult);
}
- timingTrace(doTrace, queryText, start, queryTime);
query.closeAll();
return result;
}
@@ -921,14 +923,16 @@ class MetaStoreDirectSql {
private final List<Object> params;
private final List<String> joins;
private final boolean dbHasJoinCastBug;
+ private final String defaultPartName;
- private PartitionFilterGenerator(
- Table table, List<Object> params, List<String> joins, boolean dbHasJoinCastBug) {
+ private PartitionFilterGenerator(Table table, List<Object> params, List<String> joins,
+ boolean dbHasJoinCastBug, String defaultPartName) {
this.table = table;
this.params = params;
this.joins = joins;
this.dbHasJoinCastBug = dbHasJoinCastBug;
this.filterBuffer = new FilterBuilder(false);
+ this.defaultPartName = defaultPartName;
}
/**
@@ -939,13 +943,14 @@ class MetaStoreDirectSql {
* @return the string representation of the expression tree
*/
private static String generateSqlFilter(Table table, ExpressionTree tree,
- List<Object> params, List<String> joins, boolean dbHasJoinCastBug) throws MetaException {
+ List<Object> params, List<String> joins, boolean dbHasJoinCastBug, String defaultPartName)
+ throws MetaException {
assert table != null;
if (tree.getRoot() == null) {
return "";
}
PartitionFilterGenerator visitor = new PartitionFilterGenerator(
- table, params, joins, dbHasJoinCastBug);
+ table, params, joins, dbHasJoinCastBug, defaultPartName);
tree.accept(visitor);
if (visitor.filterBuffer.hasError()) {
LOG.info("Unable to push down SQL filter: " + visitor.filterBuffer.getErrorMessage());
@@ -1071,28 +1076,33 @@ class MetaStoreDirectSql {
// Build the filter and add parameters linearly; we are traversing leaf nodes LTR.
String tableValue = "\"FILTER" + partColIndex + "\".\"PART_KEY_VAL\"";
+
if (node.isReverseOrder) {
params.add(nodeValue);
}
+ String tableColumn = tableValue;
if (colType != FilterType.String) {
// The underlying database field is varchar, we need to compare numbers.
- // Note that this won't work with __HIVE_DEFAULT_PARTITION__. It will fail and fall
- // back to JDO. That is by design; we could add an ugly workaround here but didn't.
if (colType == FilterType.Integral) {
tableValue = "cast(" + tableValue + " as decimal(21,0))";
} else if (colType == FilterType.Date) {
tableValue = "cast(" + tableValue + " as date)";
}
+ // Workaround for HIVE_DEFAULT_PARTITION - ignore it like JDO does, for now.
+ String tableValue0 = tableValue;
+ tableValue = "(case when " + tableColumn + " <> ?";
+ params.add(defaultPartName);
+
if (dbHasJoinCastBug) {
// This is a workaround for DERBY-6358 and Oracle bug; it is pretty horrible.
- tableValue = "(case when \"TBLS\".\"TBL_NAME\" = ? and \"DBS\".\"NAME\" = ? and "
+ tableValue += (" and \"TBLS\".\"TBL_NAME\" = ? and \"DBS\".\"NAME\" = ? and "
+ "\"FILTER" + partColIndex + "\".\"PART_ID\" = \"PARTITIONS\".\"PART_ID\" and "
- + "\"FILTER" + partColIndex + "\".\"INTEGER_IDX\" = " + partColIndex + " then "
- + tableValue + " else null end)";
+ + "\"FILTER" + partColIndex + "\".\"INTEGER_IDX\" = " + partColIndex);
params.add(table.getTableName().toLowerCase());
params.add(table.getDbName().toLowerCase());
}
+ tableValue += " then " + tableValue0 + " else null end)";
}
if (!node.isReverseOrder) {
params.add(nodeValue);