You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/09/25 06:57:48 UTC
svn commit: r1526106 [7/8] - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/ metastore/if/
metastore/src/gen/thrift/gen-cpp/
metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/
metastore/src/gen/thrift/gen-php/metas...
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java?rev=1526106&r1=1526105&r2=1526106&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java Wed Sep 25 04:57:46 2013
@@ -43,7 +43,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
-import org.apache.hadoop.hive.metastore.parser.FilterParser;
+import org.apache.hadoop.hive.metastore.parser.ExpressionTree.FilterBuilder;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator;
@@ -78,6 +78,9 @@ class MetaStoreDirectSql {
*/
public List<Partition> getPartitionsViaSqlFilter(
String dbName, String tblName, List<String> partNames, Integer max) throws MetaException {
+ if (partNames.isEmpty()) {
+ return new ArrayList<Partition>();
+ }
String list = repeat(",?", partNames.size()).substring(1);
return getPartitionsViaSqlFilterInternal(dbName, tblName, null,
"and PARTITIONS.PART_NAME in (" + list + ")", partNames, new ArrayList<String>(), max);
@@ -86,15 +89,18 @@ class MetaStoreDirectSql {
/**
* Gets partitions by using direct SQL queries.
* @param table The table.
- * @param parser The parsed filter from which the SQL filter will be generated.
+ * @param tree The expression tree from which the SQL filter will be derived.
* @param max The maximum number of partitions to return.
- * @return List of partitions.
+ * @return List of partitions. Null if SQL filter cannot be derived.
*/
public List<Partition> getPartitionsViaSqlFilter(
- Table table, FilterParser parser, Integer max) throws MetaException {
+ Table table, ExpressionTree tree, Integer max) throws MetaException {
+ assert tree != null;
List<String> params = new ArrayList<String>(), joins = new ArrayList<String>();
- String sqlFilter = (parser == null) ? null
- : PartitionFilterGenerator.generateSqlFilter(table, parser.tree, params, joins);
+ String sqlFilter = PartitionFilterGenerator.generateSqlFilter(table, tree, params, joins);
+ if (sqlFilter == null) {
+ return null; // Cannot make SQL filter to push down.
+ }
return getPartitionsViaSqlFilterInternal(table.getDbName(), table.getTableName(),
isViewTable(table), sqlFilter, params, joins, max);
}
@@ -135,12 +141,12 @@ class MetaStoreDirectSql {
* @param tblName Metastore table name.
* @param isView Whether table is a view. Can be passed as null if not immediately
* known, then this method will get it only if necessary.
- * @param sqlFilter SQL filter to use. Better be SQL92-compliant. Can be null.
+ * @param sqlFilter SQL filter to use. Better be SQL92-compliant.
* @param paramsForFilter params for ?-s in SQL filter text. Params must be in order.
* @param joinsForFilter if the filter needs additional join statement, they must be in
* this list. Better be SQL92-compliant.
* @param max The maximum number of partitions to return.
- * @return List of partition objects. FieldSchema is currently not populated.
+ * @return List of partition objects.
*/
private List<Partition> getPartitionsViaSqlFilterInternal(String dbName, String tblName,
Boolean isView, String sqlFilter, List<String> paramsForFilter,
@@ -163,8 +169,8 @@ class MetaStoreDirectSql {
"select PARTITIONS.PART_ID from PARTITIONS"
+ " inner join TBLS on PARTITIONS.TBL_ID = TBLS.TBL_ID "
+ " inner join DBS on TBLS.DB_ID = DBS.DB_ID "
- + join(joinsForFilter, ' ') + " where TBLS.TBL_NAME = ? and DBS.NAME = ?"
- + ((sqlFilter == null) ? "" : " " + sqlFilter) + orderForFilter;
+ + join(joinsForFilter, ' ') + " where TBLS.TBL_NAME = ? and DBS.NAME = ? "
+ + (sqlFilter == null ? "" : sqlFilter) + orderForFilter;
Object[] params = new Object[paramsForFilter.size() + 2];
params[0] = tblName;
params[1] = dbName;
@@ -539,17 +545,18 @@ class MetaStoreDirectSql {
return rv;
}
- private static class PartitionFilterGenerator implements TreeVisitor {
+ private static class PartitionFilterGenerator extends TreeVisitor {
private final Table table;
- private final StringBuilder filterBuffer;
+ private final FilterBuilder filterBuffer;
private final List<String> params;
private final List<String> joins;
- private PartitionFilterGenerator(Table table, List<String> params, List<String> joins) {
+ private PartitionFilterGenerator(
+ Table table, List<String> params, List<String> joins) {
this.table = table;
this.params = params;
this.joins = joins;
- this.filterBuffer = new StringBuilder();
+ this.filterBuffer = new FilterBuilder(false);
}
/**
@@ -566,37 +573,53 @@ class MetaStoreDirectSql {
return "";
}
PartitionFilterGenerator visitor = new PartitionFilterGenerator(table, params, joins);
- tree.getRoot().accept(visitor);
+ tree.accept(visitor);
+ if (visitor.filterBuffer.hasError()) {
+ LOG.info("Unable to push down SQL filter: " + visitor.filterBuffer.getErrorMessage());
+ return null;
+ }
+
// Some joins might be null (see processNode for LeafNode), clean them up.
for (int i = 0; i < joins.size(); ++i) {
if (joins.get(i) != null) continue;
joins.remove(i--);
}
- return "and (" + visitor.filterBuffer.toString() + ")";
+ return "and (" + visitor.filterBuffer.getFilter() + ")";
+ }
+
+ @Override
+ protected void beginTreeNode(TreeNode node) throws MetaException {
+ filterBuffer.append(" (");
}
@Override
- public void visit(TreeNode node) throws MetaException {
- assert node != null && node.getLhs() != null && node.getRhs() != null;
- filterBuffer.append (" (");
- node.getLhs().accept(this);
+ protected void midTreeNode(TreeNode node) throws MetaException {
filterBuffer.append((node.getAndOr() == LogicalOperator.AND) ? " and " : " or ");
- node.getRhs().accept(this);
- filterBuffer.append (") ");
+ }
+
+ @Override
+ protected void endTreeNode(TreeNode node) throws MetaException {
+ filterBuffer.append(") ");
+ }
+
+ @Override
+ protected boolean shouldStop() {
+ return filterBuffer.hasError();
}
@Override
public void visit(LeafNode node) throws MetaException {
if (node.operator == Operator.LIKE) {
- // ANSI92 supports || for concatenation (we need to concat '%'-s to the parameter),
- // but it doesn't work on all RDBMSes, e.g. on MySQL by default. So don't use it for now.
- throw new MetaException("LIKE is not supported for SQL filter pushdown");
+ filterBuffer.setError("LIKE is not supported for SQL filter pushdown");
+ return;
}
int partColCount = table.getPartitionKeys().size();
- int partColIndex = node.getPartColIndexForFilter(table);
+ int partColIndex = node.getPartColIndexForFilter(table, filterBuffer);
+ if (filterBuffer.hasError()) return;
- String valueAsString = node.getFilterPushdownParam(table, partColIndex);
// Add parameters linearly; we are traversing leaf nodes LTR, so they would match correctly.
+ String valueAsString = node.getFilterPushdownParam(table, partColIndex, filterBuffer);
+ if (filterBuffer.hasError()) return;
params.add(valueAsString);
if (joins.isEmpty()) {
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1526106&r1=1526105&r2=1526106&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Wed Sep 25 04:57:46 2013
@@ -28,6 +28,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -55,6 +56,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -120,10 +122,17 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.model.MTablePrivilege;
import org.apache.hadoop.hive.metastore.model.MType;
import org.apache.hadoop.hive.metastore.model.MVersionTable;
+import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream;
+import org.apache.hadoop.hive.metastore.parser.ExpressionTree.FilterBuilder;
+import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode;
+import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator;
+import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode;
+import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor;
import org.apache.hadoop.hive.metastore.parser.FilterLexer;
import org.apache.hadoop.hive.metastore.parser.FilterParser;
import org.apache.hadoop.util.StringUtils;
+import org.apache.thrift.TException;
import org.datanucleus.store.rdbms.exceptions.MissingTableException;
/**
@@ -161,6 +170,7 @@ public class ObjectStore implements RawS
private boolean isInitialized = false;
private PersistenceManager pm = null;
private MetaStoreDirectSql directSql = null;
+ private PartitionExpressionProxy expressionProxy = null;
private Configuration hiveConf;
int openTrasactionCalls = 0;
private Transaction currentTransaction = null;
@@ -202,6 +212,7 @@ public class ObjectStore implements RawS
// most recent instance of the pmf
pm = null;
directSql = null;
+ expressionProxy = null;
openTrasactionCalls = 0;
currentTransaction = null;
transactionStatus = TXN_STATUS.NO_STATE;
@@ -234,9 +245,30 @@ public class ObjectStore implements RawS
pm = getPersistenceManager();
isInitialized = pm != null;
if (isInitialized) {
+ expressionProxy = createExpressionProxy(hiveConf);
directSql = new MetaStoreDirectSql(pm);
}
- return;
+ }
+
+ /**
+ * Creates the proxy used to evaluate expressions. This is here to prevent circular
+ * dependency - ql -> metastore client <-> metastore server -> ql. If server and
+ * client are split, this can be removed.
+ * @param conf Configuration.
+ * @return The partition expression proxy.
+ */
+ private static PartitionExpressionProxy createExpressionProxy(Configuration conf) {
+ String className = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS);
+ try {
+ @SuppressWarnings("unchecked")
+ Class<? extends PartitionExpressionProxy> clazz =
+ (Class<? extends PartitionExpressionProxy>)MetaStoreUtils.getClass(className);
+ return (PartitionExpressionProxy)MetaStoreUtils.newInstance(
+ clazz, new Class<?>[0], new Object[0]);
+ } catch (MetaException e) {
+ LOG.error("Error loading PartitionExpressionProxy", e);
+ throw new RuntimeException("Error loading PartitionExpressionProxy: " + e.getMessage());
+ }
}
/**
@@ -1493,13 +1525,22 @@ public class ObjectStore implements RawS
}
- private List<Partition> convertToParts(List<MPartition> mparts)
+ private List<Partition> convertToParts(List<MPartition> mparts) throws MetaException {
+ return convertToParts(mparts, null);
+ }
+
+ private List<Partition> convertToParts(List<MPartition> src, List<Partition> dest)
throws MetaException {
- List<Partition> parts = new ArrayList<Partition>(mparts.size());
- for (MPartition mp : mparts) {
- parts.add(convertToPart(mp));
+ if (src == null) {
+ return dest;
}
- return parts;
+ if (dest == null) {
+ dest = new ArrayList<Partition>(src.size());
+ }
+ for (MPartition mp : src) {
+ dest.add(convertToPart(mp));
+ }
+ return dest;
}
private List<Partition> convertToParts(String dbName, String tblName, List<MPartition> mparts)
@@ -1514,27 +1555,12 @@ public class ObjectStore implements RawS
// TODO:pc implement max
public List<String> listPartitionNames(String dbName, String tableName,
short max) throws MetaException {
- List<String> pns = new ArrayList<String>();
+ List<String> pns = null;
boolean success = false;
try {
openTransaction();
LOG.debug("Executing getPartitionNames");
- dbName = dbName.toLowerCase().trim();
- tableName = tableName.toLowerCase().trim();
- Query q = pm.newQuery(
- "select partitionName from org.apache.hadoop.hive.metastore.model.MPartition "
- + "where table.database.name == t1 && table.tableName == t2 "
- + "order by partitionName asc");
- q.declareParameters("java.lang.String t1, java.lang.String t2");
- q.setResult("partitionName");
-
- if(max > 0) {
- q.setRange(0, max);
- }
- Collection names = (Collection) q.execute(dbName, tableName);
- for (Iterator i = names.iterator(); i.hasNext();) {
- pns.add((String) i.next());
- }
+ pns = getPartitionNamesNoTxn(dbName, tableName, max);
success = commitTransaction();
} finally {
if (!success) {
@@ -1544,6 +1570,27 @@ public class ObjectStore implements RawS
return pns;
}
+ private List<String> getPartitionNamesNoTxn(String dbName, String tableName, short max) {
+ List<String> pns = new ArrayList<String>();
+ dbName = dbName.toLowerCase().trim();
+ tableName = tableName.toLowerCase().trim();
+ Query q = pm.newQuery(
+ "select partitionName from org.apache.hadoop.hive.metastore.model.MPartition "
+ + "where table.database.name == t1 && table.tableName == t2 "
+ + "order by partitionName asc");
+ q.declareParameters("java.lang.String t1, java.lang.String t2");
+ q.setResult("partitionName");
+
+ if(max > 0) {
+ q.setRange(0, max);
+ }
+ Collection names = (Collection) q.execute(dbName, tableName);
+ for (Iterator i = names.iterator(); i.hasNext();) {
+ pns.add((String) i.next());
+ }
+ return pns;
+ }
+
/**
* Retrieves a Collection of partition-related results from the database that match
* the partial specification given for a specific table.
@@ -1704,6 +1751,8 @@ public class ObjectStore implements RawS
List<String> partNames, boolean allowSql, boolean allowJdo)
throws MetaException, NoSuchObjectException {
assert allowSql || allowJdo;
+ dbName = dbName.toLowerCase();
+ tblName = tblName.toLowerCase();
boolean doTrace = LOG.isDebugEnabled();
boolean doUseDirectSql = canUseDirectSql(allowSql);
@@ -1723,19 +1772,224 @@ public class ObjectStore implements RawS
}
if (!doUseDirectSql) {
- results = getPartitionsViaOrm(dbName, tblName, partNames);
+ results = getPartitionsViaOrmFilter(dbName, tblName, partNames);
}
success = commitTransaction();
if (doTrace) {
LOG.debug(results.size() + " partition retrieved using " + (doUseDirectSql ? "SQL" : "ORM")
+ " in " + ((System.nanoTime() - start) / 1000000.0) + "ms");
}
- return results;
} finally {
if (!success) {
rollbackTransaction();
}
}
+ return results;
+ }
+
+ @Override
+ public boolean getPartitionsByExpr(String dbName, String tblName, byte[] expr,
+ String defaultPartitionName, short maxParts, Set<Partition> result) throws TException {
+ return getPartitionsByExprInternal(
+ dbName, tblName, expr, defaultPartitionName, maxParts, result, true, true);
+ }
+
+ protected boolean getPartitionsByExprInternal(String dbName, String tblName,
+ byte[] expr, String defaultPartitionName, short maxParts, Set<Partition> result,
+ boolean allowSql, boolean allowJdo) throws TException {
+ assert allowSql || allowJdo;
+ assert result != null;
+ dbName = dbName.toLowerCase();
+ tblName = tblName.toLowerCase();
+
+ // We will try pushdown first, so make the filter. This will also validate the expression,
+ // if serialization fails we will throw incompatible metastore error to the client.
+ String filter = null;
+ try {
+ filter = expressionProxy.convertExprToFilter(expr);
+ } catch (MetaException ex) {
+ throw new IMetaStoreClient.IncompatibleMetastoreException(ex.getMessage());
+ }
+
+ // Make a tree out of the filter.
+ // TODO: this is all pretty ugly. The only reason we need all these transformations
+ // is to maintain support for simple filters for HCat users that query metastore.
+ // If forcing everyone to use thick client is out of the question, maybe we could
+ // parse the filter into standard hive expressions and not all this separate tree
+ // Filter.g stuff. That way this method and ...ByFilter would just be merged.
+ ExpressionTree exprTree = makeExpressionTree(filter);
+
+ boolean doUseDirectSql = allowSql && isDirectSqlEnabled(maxParts);
+ boolean doTrace = LOG.isDebugEnabled();
+ List<Partition> partitions = null;
+ boolean hasUnknownPartitions = false;
+ boolean success = false;
+ try {
+ long start = doTrace ? System.nanoTime() : 0;
+ openTransaction();
+ Table table = ensureGetTable(dbName, tblName);
+ if (doUseDirectSql) {
+ try {
+ if (exprTree != null) {
+ // We have some sort of expression tree, try SQL filter pushdown.
+ partitions = directSql.getPartitionsViaSqlFilter(table, exprTree, null);
+ }
+ if (partitions == null) {
+ // We couldn't do SQL filter pushdown. Get names via normal means.
+ List<String> partNames = new LinkedList<String>();
+ hasUnknownPartitions = getPartitionNamesPrunedByExprNoTxn(
+ table, expr, defaultPartitionName, maxParts, partNames);
+ partitions = directSql.getPartitionsViaSqlFilter(dbName, tblName, partNames, null);
+ }
+ } catch (Exception ex) {
+ handleDirectSqlError(allowJdo, ex);
+ doUseDirectSql = false;
+ table = ensureGetTable(dbName, tblName); // Get again, detached on rollback.
+ }
+ }
+
+ if (!doUseDirectSql) {
+ assert partitions == null;
+ if (exprTree != null) {
+ // We have some sort of expression tree, try JDOQL filter pushdown.
+ partitions = getPartitionsViaOrmFilter(table, exprTree, maxParts, false);
+ }
+ if (partitions == null) {
+ // We couldn't do JDOQL filter pushdown. Get names via normal means.
+ List<String> partNames = new ArrayList<String>();
+ hasUnknownPartitions = getPartitionNamesPrunedByExprNoTxn(
+ table, expr, defaultPartitionName, maxParts, partNames);
+ partitions = getPartitionsViaOrmFilter(dbName, tblName, partNames);
+ }
+ }
+ success = commitTransaction();
+ if (doTrace) {
+ double time = ((System.nanoTime() - start) / 1000000.0);
+ LOG.debug(partitions.size() + " partition retrieved using "
+ + (doUseDirectSql ? "SQL" : "ORM") + " in " + time + "ms");
+ }
+ } finally {
+ if (!success) {
+ rollbackTransaction();
+ }
+ }
+ result.addAll(partitions);
+ return hasUnknownPartitions;
+ }
+
+ private boolean isDirectSqlEnabled(short maxParts) {
+ // There's no portable SQL limit. It doesn't make a lot of sense w/o offset anyway.
+ return (maxParts < 0)
+ && HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL);
+ }
+
+ private class LikeChecker extends ExpressionTree.TreeVisitor {
+ private boolean hasLike;
+
+ public boolean hasLike() {
+ return hasLike;
+ }
+
+ @Override
+ protected boolean shouldStop() {
+ return hasLike;
+ }
+
+ @Override
+ protected void visit(LeafNode node) throws MetaException {
+ hasLike = hasLike || (node.operator == Operator.LIKE);
+ }
+ }
+
+ /**
+ * Makes expression tree out of expr.
+ * @param filter Filter.
+ * @return Expression tree. Null if there was an error.
+ */
+ private ExpressionTree makeExpressionTree(String filter) throws MetaException {
+ // TODO: ExprNodeDesc is an expression tree, we could just use that and be rid of Filter.g.
+ if (filter == null || filter.isEmpty()) {
+ return ExpressionTree.EMPTY_TREE;
+ }
+ LOG.debug("Filter specified is " + filter);
+ ExpressionTree tree = null;
+ try {
+ tree = getFilterParser(filter).tree;
+ } catch (MetaException ex) {
+ LOG.info("Unable to make the expression tree from expression string ["
+ + filter + "]" + ex.getMessage()); // Don't log the stack, this is normal.
+ }
+ if (tree == null) {
+ return null;
+ }
+ // We suspect that LIKE pushdown into JDO is invalid; see HIVE-5134. Check for like here.
+ LikeChecker lc = new LikeChecker();
+ tree.accept(lc);
+ return lc.hasLike() ? null : tree;
+ }
+
+ /**
+ * Gets the partition names from a table, pruned using an expression.
+ * @param table Table.
+ * @param expr Expression.
+ * @param defaultPartName Default partition name from job config, if any.
+ * @param maxParts Maximum number of partition names to return.
+ * @param result The resulting names.
+ * @return Whether the result contains any unknown partitions.
+ */
+ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr,
+ String defaultPartName, short maxParts, List<String> result) throws MetaException {
+ result.addAll(getPartitionNamesNoTxn(
+ table.getDbName(), table.getTableName(), maxParts));
+ List<String> columnNames = new ArrayList<String>();
+ for (FieldSchema fs : table.getPartitionKeys()) {
+ columnNames.add(fs.getName());
+ }
+ if (defaultPartName == null || defaultPartName.isEmpty()) {
+ defaultPartName = HiveConf.getVar(getConf(), HiveConf.ConfVars.DEFAULTPARTITIONNAME);
+ }
+ return expressionProxy.filterPartitionsByExpr(
+ columnNames, expr, defaultPartName, result);
+ }
+
+ /**
+ * Gets partition names from the table via ORM (JDOQL) filter pushdown.
+ * @param table The table.
+ * @param tree The expression tree from which JDOQL filter will be made.
+ * @param maxParts Maximum number of partitions to return.
+ * @param isValidatedFilter Whether the filter was pre-validated for JDOQL pushdown by a client
+ * (old hive client or non-hive one); if it was and we fail to create a filter, we will throw.
+ * @return Resulting partitions. Can be null if isValidatedFilter is false, and
+ * there was error deriving the JDO filter.
+ */
+ private List<Partition> getPartitionsViaOrmFilter(Table table, ExpressionTree tree,
+ short maxParts, boolean isValidatedFilter) throws MetaException {
+ Map<String, Object> params = new HashMap<String, Object>();
+ String jdoFilter = makeQueryFilterString(
+ table.getDbName(), table, tree, params, isValidatedFilter);
+ if (jdoFilter == null) {
+ assert !isValidatedFilter;
+ return null;
+ }
+ Query query = pm.newQuery(MPartition.class, jdoFilter);
+ if (maxParts >= 0) {
+ // User specified a row limit, set it on the Query
+ query.setRange(0, maxParts);
+ }
+
+ String parameterDeclaration = makeParameterDeclarationStringObj(params);
+ query.declareParameters(parameterDeclaration);
+ query.setOrdering("partitionName ascending");
+
+ @SuppressWarnings("unchecked")
+ List<MPartition> mparts = (List<MPartition>) query.executeWithMap(params);
+
+ LOG.debug("Done executing query for getPartitionsViaOrmFilter");
+ pm.retrieveAll(mparts); // TODO: why is this inconsistent with what we get by names?
+ LOG.debug("Done retrieving all objects for getPartitionsViaOrmFilter");
+ List<Partition> results = convertToParts(mparts);
+ query.closeAll();
+ return results;
}
private void handleDirectSqlError(boolean allowJdo, Exception ex) throws MetaException {
@@ -1750,8 +2004,18 @@ public class ObjectStore implements RawS
openTransaction();
}
- private List<Partition> getPartitionsViaOrm(
+ /**
+ * Gets partition names from the table via ORM (JDOQL) name filter.
+ * @param dbName Database name.
+ * @param tblName Table name.
+ * @param partNames Partition names to get the objects for.
+ * @return Resulting partitions.
+ */
+ private List<Partition> getPartitionsViaOrmFilter(
String dbName, String tblName, List<String> partNames) throws MetaException {
+ if (partNames.isEmpty()) {
+ return new ArrayList<Partition>();
+ }
StringBuilder sb = new StringBuilder(
"table.tableName == t1 && table.database.name == t2 && (");
int n = 0;
@@ -1778,6 +2042,7 @@ public class ObjectStore implements RawS
query.declareParameters(parameterDeclaration);
query.setOrdering("partitionName ascending");
+ @SuppressWarnings("unchecked")
List<MPartition> mparts = (List<MPartition>) query.executeWithMap(params);
// pm.retrieveAll(mparts); // retrieveAll is pessimistic. some fields may not be needed
List<Partition> results = convertToParts(dbName, tblName, mparts);
@@ -1798,39 +2063,43 @@ public class ObjectStore implements RawS
assert allowSql || allowJdo;
boolean doTrace = LOG.isDebugEnabled();
boolean doUseDirectSql = canUseDirectSql(allowSql);
+
dbName = dbName.toLowerCase();
tblName = tblName.toLowerCase();
- List<Partition> results = null;
- FilterParser parser = null;
- if (filter != null && filter.length() != 0) {
- LOG.debug("Filter specified is " + filter);
- parser = getFilterParser(filter);
- }
+ ExpressionTree tree = (filter != null && !filter.isEmpty())
+ ? getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE;
+ List<Partition> results = null;
boolean success = false;
try {
long start = doTrace ? System.nanoTime() : 0;
openTransaction();
- MTable mtable = ensureGetMTable(dbName, tblName);
+ Table table = ensureGetTable(dbName, tblName);
if (doUseDirectSql) {
try {
- Table table = convertToTable(mtable);
Integer max = (maxParts < 0) ? null : (int)maxParts;
- results = directSql.getPartitionsViaSqlFilter(table, parser, max);
+ results = directSql.getPartitionsViaSqlFilter(table, tree, max);
+ if (results == null) {
+ // Cannot push down SQL filter. The message has been logged internally.
+ // This is not an error so don't roll back, just go to JDO.
+ doUseDirectSql = false;
+ }
} catch (Exception ex) {
handleDirectSqlError(allowJdo, ex);
doUseDirectSql = false;
start = doTrace ? System.nanoTime() : 0;
- mtable = ensureGetMTable(dbName, tblName); // detached on rollback, get again
+ table = ensureGetTable(dbName, tblName); // detached on rollback, get again
}
}
+
if (!doUseDirectSql) {
- results = convertToParts(listMPartitionsByFilterNoTxn(
- mtable, dbName, tblName, parser, maxParts));
+ results = getPartitionsViaOrmFilter(table, tree, maxParts, true);
}
success = commitTransaction();
- LOG.info(results.size() + " partitions retrieved using " + (doUseDirectSql ? "SQL" : "ORM")
- + (doTrace ? (" in " + ((System.nanoTime() - start) / 1000000.0) + "ms") : ""));
+ if (doTrace) {
+ LOG.debug(results.size() + " partition retrieved using " + (doUseDirectSql ? "SQL" : "ORM")
+ + " in " + ((System.nanoTime() - start) / 1000000.0) + "ms");
+ }
return results;
} finally {
if (!success) {
@@ -1849,13 +2118,20 @@ public class ObjectStore implements RawS
&& !isActiveTransaction();
}
- private MTable ensureGetMTable(String dbName, String tblName) throws NoSuchObjectException {
+ /**
+ * Gets the table object for a given table, throws if anything goes wrong.
+ * @param dbName Database name.
+ * @param tblName Table name.
+ * @return Table object.
+ */
+ private Table ensureGetTable(
+ String dbName, String tblName) throws NoSuchObjectException, MetaException {
MTable mtable = getMTable(dbName, tblName);
if (mtable == null) {
throw new NoSuchObjectException("Specified database/table does not exist : "
+ dbName + "." + tblName);
}
- return mtable;
+ return convertToTable(mtable);
}
private FilterParser getFilterParser(String filter) throws MetaException {
@@ -1881,56 +2157,55 @@ public class ObjectStore implements RawS
}
/**
- * Makes a JDO query filter string
- * if mtable is not null, generates the query to filter over partitions in a table.
- * if mtable is null, generates the query to filter over tables in a database
+ * Makes a JDO query filter string.
+ * Makes a JDO query filter string for tables or partitions.
+ * @param dbName Database name.
+ * @param table Table. If null, the query returned is over tables in a database.
+ * If not null, the query returned is over partitions in a table.
+ * @param filter The filter from which JDOQL filter will be made.
+ * @param params Parameters for the filter. Some parameters may be added here.
+ * @return Resulting filter.
*/
- private String makeQueryFilterString(MTable mtable, String filter,
+ private String makeQueryFilterString(String dbName, MTable mtable, String filter,
Map<String, Object> params) throws MetaException {
- FilterParser parser =
- (filter != null && filter.length() != 0) ? getFilterParser(filter) : null;
- return makeQueryFilterString(mtable, parser, params);
+ ExpressionTree tree = (filter != null && !filter.isEmpty())
+ ? getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE;
+ return makeQueryFilterString(dbName, convertToTable(mtable), tree, params, true);
}
/**
- * Makes a JDO query filter string
- * if mtable is not null, generates the query to filter over partitions in a table.
- * if mtable is null, generates the query to filter over tables in a database
+ * Makes a JDO query filter string for tables or partitions.
+ * @param dbName Database name.
+ * @param table Table. If null, the query returned is over tables in a database.
+ * If not null, the query returned is over partitions in a table.
+ * @param tree The expression tree from which JDOQL filter will be made.
+ * @param params Parameters for the filter. Some parameters may be added here.
+ * @param isValidatedFilter Whether the filter was pre-validated for JDOQL pushdown
+ * by the client; if it was and we fail to create a filter, we will throw.
+ * @return Resulting filter. Can be null if isValidatedFilter is false, and there was error.
*/
- private String makeQueryFilterString(MTable mtable, FilterParser parser,
- Map<String, Object> params)
- throws MetaException {
-
- StringBuilder queryBuilder = new StringBuilder();
- if (mtable != null) {
+ private String makeQueryFilterString(String dbName, Table table, ExpressionTree tree,
+ Map<String, Object> params, boolean isValidatedFilter) throws MetaException {
+ assert tree != null;
+ FilterBuilder queryBuilder = new FilterBuilder(isValidatedFilter);
+ if (table != null) {
queryBuilder.append("table.tableName == t1 && table.database.name == t2");
+ params.put("t1", table.getTableName());
+ params.put("t2", table.getDbName());
} else {
queryBuilder.append("database.name == dbName");
+ params.put("dbName", dbName);
}
- if (parser != null) {
- String jdoFilter;
- if (mtable != null) {
- Table table = convertToTable(mtable);
- jdoFilter = parser.tree.generateJDOFilter(table, params);
- } else {
- jdoFilter = parser.tree.generateJDOFilter(null, params);
- }
- LOG.debug("jdoFilter = " + jdoFilter);
-
- if( jdoFilter.trim().length() > 0 ) {
- queryBuilder.append(" && ( ");
- queryBuilder.append(jdoFilter.trim());
- queryBuilder.append(" )");
- }
+ tree.generateJDOFilterFragment(table, params, queryBuilder);
+ if (queryBuilder.hasError()) {
+ assert !isValidatedFilter;
+ LOG.info("JDO filter pushdown cannot be used: " + queryBuilder.getErrorMessage());
+ return null;
}
- return queryBuilder.toString();
- }
-
- private String makeTableQueryFilterString(String filter,
- Map<String, Object> params)
- throws MetaException {
- return makeQueryFilterString(null, filter, params);
+ String jdoFilter = queryBuilder.getFilter();
+ LOG.debug("jdoFilter = " + jdoFilter);
+ return jdoFilter;
}
private String makeParameterDeclarationString(Map<String, String> params) {
@@ -1954,38 +2229,6 @@ public class ObjectStore implements RawS
return paramDecl.toString();
}
- private List<MPartition> listMPartitionsByFilterNoTxn(MTable mtable, String dbName,
- String tableName, FilterParser parser, short maxParts)
- throws MetaException, NoSuchObjectException {
- List<MPartition> mparts = null;
- LOG.debug("Executing listMPartitionsByFilterNoTxn");
- Map<String, Object> params = new HashMap<String, Object>();
- String queryFilterString = makeQueryFilterString(mtable, parser, params);
-
- Query query = pm.newQuery(MPartition.class,
- queryFilterString);
-
- if( maxParts >= 0 ) {
- //User specified a row limit, set it on the Query
- query.setRange(0, maxParts);
- }
-
- LOG.debug("JDOQL filter is " + queryFilterString);
-
- params.put("t1", tableName.trim());
- params.put("t2", dbName.trim());
-
- String parameterDeclaration = makeParameterDeclarationStringObj(params);
- query.declareParameters(parameterDeclaration);
- query.setOrdering("partitionName ascending");
-
- mparts = (List<MPartition>) query.executeWithMap(params);
-
- LOG.debug("Done executing query for listMPartitionsByFilterNoTxn");
- pm.retrieveAll(mparts);
- return mparts;
- }
-
@Override
public List<String> listTableNamesByFilter(String dbName, String filter, short maxTables)
throws MetaException {
@@ -1996,7 +2239,7 @@ public class ObjectStore implements RawS
LOG.debug("Executing listTableNamesByFilter");
dbName = dbName.toLowerCase().trim();
Map<String, Object> params = new HashMap<String, Object>();
- String queryFilterString = makeTableQueryFilterString(filter, params);
+ String queryFilterString = makeQueryFilterString(dbName, null, filter, params);
Query query = pm.newQuery(MTable.class);
query.declareImports("import java.lang.String");
query.setResult("tableName");
@@ -2005,7 +2248,6 @@ public class ObjectStore implements RawS
query.setRange(0, maxTables);
}
LOG.debug("filter specified is " + filter + "," + " JDOQL filter is " + queryFilterString);
- params.put("dbName", dbName);
for (Entry<String, Object> entry : params.entrySet()) {
LOG.debug("key: " + entry.getKey() + " value: " + entry.getValue() +
" class: " + entry.getValue().getClass().getName());
@@ -2050,7 +2292,7 @@ public class ObjectStore implements RawS
return partNames;
}
Map<String, Object> params = new HashMap<String, Object>();
- String queryFilterString = makeQueryFilterString(mtable, filter, params);
+ String queryFilterString = makeQueryFilterString(dbName, mtable, filter, params);
Query query = pm.newQuery(
"select partitionName from org.apache.hadoop.hive.metastore.model.MPartition "
+ "where " + queryFilterString);
@@ -2064,9 +2306,6 @@ public class ObjectStore implements RawS
" JDOQL filter is " + queryFilterString);
LOG.debug("Parms is " + params);
- params.put("t1", tableName.trim());
- params.put("t2", dbName.trim());
-
String parameterDeclaration = makeParameterDeclarationStringObj(params);
query.declareParameters(parameterDeclaration);
query.setOrdering("partitionName ascending");
Added: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java?rev=1526106&view=auto
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java (added)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java Wed Sep 25 04:57:46 2013
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+/**
+ * The proxy interface that metastore uses to manipulate and apply
+ * serialized filter expressions coming from client.
+ */
+public interface PartitionExpressionProxy {
+
+ /**
+ * Converts serialized Hive expression into filter in the format suitable for Filter.g.
+ * @param expr Serialized expression.
+ * @return The filter string.
+ */
+ public String convertExprToFilter(byte[] expr) throws MetaException;
+
+ /**
+ * Filters the partition names via serialized Hive expression.
+ * @param columnNames Partition column names in the underlying table.
+ * @param expr Serialized expression.
+ * @param defaultPartitionName Default partition name from job or server configuration.
+ * @param partitionNames Partition names; the list is modified in place.
+ * @return Whether there were any unknown partitions preserved in the name list.
+ */
+ public boolean filterPartitionsByExpr(List<String> columnNames, byte[] expr,
+ String defaultPartitionName, List<String> partitionNames) throws MetaException;
+}
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java?rev=1526106&r1=1526105&r2=1526106&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java Wed Sep 25 04:57:46 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.metastore
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
@@ -48,6 +49,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.model.MRoleMap;
import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege;
import org.apache.hadoop.hive.metastore.model.MTablePrivilege;
+import org.apache.thrift.TException;
public interface RawStore extends Configurable {
@@ -186,6 +188,10 @@ public interface RawStore extends Config
String dbName, String tblName, String filter, short maxParts)
throws MetaException, NoSuchObjectException;
+ public abstract boolean getPartitionsByExpr(String dbName, String tblName,
+ byte[] expr, String defaultPartitionName, short maxParts, Set<Partition> result)
+ throws TException;
+
public abstract List<Partition> getPartitionsByNames(
String dbName, String tblName, List<String> partNames)
throws MetaException, NoSuchObjectException;
@@ -441,5 +447,4 @@ public interface RawStore extends Config
public abstract String getMetaStoreSchemaVersion() throws MetaException;
public abstract void setMetaStoreSchemaVersion(String version, String comment) throws MetaException;
-
}
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java?rev=1526106&r1=1526105&r2=1526106&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java Wed Sep 25 04:57:46 2013
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.metastore.parser;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Stack;
@@ -26,11 +25,12 @@ import java.util.Stack;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CharStream;
import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.metastore.ObjectStore;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.serde.serdeConstants;
import com.google.common.collect.Sets;
@@ -39,6 +39,8 @@ import com.google.common.collect.Sets;
* at intermediate level and the leaf level nodes are of type LeafNode.
*/
public class ExpressionTree {
+ /** The empty tree that can be returned for an empty filter. */
+ public static final ExpressionTree EMPTY_TREE = new ExpressionTree();
/** The logical operations supported. */
public enum LogicalOperator {
@@ -104,9 +106,81 @@ public class ExpressionTree {
}
- public static interface TreeVisitor {
- void visit(TreeNode node) throws MetaException;
- void visit(LeafNode node) throws MetaException;
+ /**
+ * Depth first traversal of ExpressionTree.
+ * The users should override the subset of methods to do their stuff.
+ */
+ public static class TreeVisitor {
+ private void visit(TreeNode node) throws MetaException {
+ if (shouldStop()) return;
+ assert node != null && node.getLhs() != null && node.getRhs() != null;
+ beginTreeNode(node);
+ node.lhs.accept(this);
+ midTreeNode(node);
+ node.rhs.accept(this);
+ endTreeNode(node);
+ }
+
+ protected void beginTreeNode(TreeNode node) throws MetaException {}
+ protected void midTreeNode(TreeNode node) throws MetaException {}
+ protected void endTreeNode(TreeNode node) throws MetaException {}
+ protected void visit(LeafNode node) throws MetaException {}
+ protected boolean shouldStop() {
+ return false;
+ }
+ }
+
+ /**
+ * Helper class that wraps the stringbuilder used to build the filter over the tree,
+ * as well as error propagation in two modes - expect errors, i.e. filter might as well
+ * be unbuildable and that's not a failure condition; or don't expect errors, i.e. filter
+ * must be buildable.
+ */
+ public static class FilterBuilder {
+ private final StringBuilder result = new StringBuilder();
+ private String errorMessage = null;
+ private boolean expectNoErrors = false;
+
+ public FilterBuilder(boolean expectNoErrors) {
+ this.expectNoErrors = expectNoErrors;
+ }
+
+ public String getFilter() throws MetaException {
+ assert errorMessage == null;
+ if (errorMessage != null) {
+ throw new MetaException("Trying to get result after error: " + errorMessage);
+ }
+ return result.toString();
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return getFilter();
+ } catch (MetaException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+
+ public boolean hasError() {
+ return errorMessage != null;
+ }
+
+ public FilterBuilder append(String filterPart) {
+ this.result.append(filterPart);
+ return this;
+ }
+
+ public void setError(String errorMessage) throws MetaException {
+ this.errorMessage = errorMessage;
+ if (expectNoErrors) {
+ throw new MetaException(errorMessage);
+ }
+ }
}
/**
@@ -139,7 +213,7 @@ public class ExpressionTree {
}
/** Double dispatch for TreeVisitor. */
- public void accept(TreeVisitor visitor) throws MetaException {
+ protected void accept(TreeVisitor visitor) throws MetaException {
visitor.visit(this);
}
@@ -153,16 +227,16 @@ public class ExpressionTree {
* tables that match the filter.
* @param params
* A map of parameter key to values for the filter statement.
+ * @param filterBuilder The filter builder that is used to build filter.
* @return a JDO filter statement
* @throws MetaException
*/
- public String generateJDOFilter(Table table, Map<String, Object> params)
- throws MetaException {
- StringBuilder filterBuffer = new StringBuilder();
-
- if ( lhs != null) {
+ public void generateJDOFilter(Table table, Map<String, Object> params,
+ FilterBuilder filterBuffer) throws MetaException {
+ if (filterBuffer.hasError()) return;
+ if (lhs != null) {
filterBuffer.append (" (");
- filterBuffer.append(lhs.generateJDOFilter(table, params));
+ lhs.generateJDOFilter(table, params, filterBuffer);
if (rhs != null) {
if( andOr == LogicalOperator.AND ) {
@@ -171,12 +245,10 @@ public class ExpressionTree {
filterBuffer.append(" || ");
}
- filterBuffer.append(rhs.generateJDOFilter(table, params));
+ rhs.generateJDOFilter(table, params, filterBuffer);
}
filterBuffer.append (") ");
}
-
- return filterBuffer.toString();
}
}
@@ -192,18 +264,17 @@ public class ExpressionTree {
private static final String PARAM_PREFIX = "hive_filter_param_";
@Override
- public void accept(TreeVisitor visitor) throws MetaException {
+ protected void accept(TreeVisitor visitor) throws MetaException {
visitor.visit(this);
}
@Override
- public String generateJDOFilter(Table table,
- Map<String, Object> params)
- throws MetaException {
+ public void generateJDOFilter(Table table, Map<String, Object> params,
+ FilterBuilder filterBuilder) throws MetaException {
if (table != null) {
- return generateJDOFilterOverPartitions(table, params);
+ generateJDOFilterOverPartitions(table, params, filterBuilder);
} else {
- return generateJDOFilterOverTables(params);
+ generateJDOFilterOverTables(params, filterBuilder);
}
}
@@ -212,20 +283,22 @@ public class ExpressionTree {
private static final Set<Operator> TABLE_FILTER_OPS = Sets.newHashSet(
Operator.EQUALS, Operator.NOTEQUALS, Operator.NOTEQUALS2);
- private String generateJDOFilterOverTables(Map<String, Object> params)
- throws MetaException {
+ private void generateJDOFilterOverTables(Map<String, Object> params,
+ FilterBuilder filterBuilder) throws MetaException {
if (keyName.equals(hive_metastoreConstants.HIVE_FILTER_FIELD_OWNER)) {
keyName = "this.owner";
} else if (keyName.equals(hive_metastoreConstants.HIVE_FILTER_FIELD_LAST_ACCESS)) {
//lastAccessTime expects an integer, so we cannot use the "like operator"
if (operator == Operator.LIKE) {
- throw new MetaException("Like is not supported for HIVE_FILTER_FIELD_LAST_ACCESS");
+ filterBuilder.setError("Like is not supported for HIVE_FILTER_FIELD_LAST_ACCESS");
+ return;
}
keyName = "this.lastAccessTime";
} else if (keyName.startsWith(hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS)) {
if (!TABLE_FILTER_OPS.contains(operator)) {
- throw new MetaException("Only " + TABLE_FILTER_OPS + " are supported " +
+ filterBuilder.setError("Only " + TABLE_FILTER_OPS + " are supported " +
"operators for HIVE_FILTER_FIELD_PARAMS");
+ return;
}
String paramKeyName = keyName.substring(hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS.length());
keyName = "this.parameters.get(\"" + paramKeyName + "\")";
@@ -233,10 +306,11 @@ public class ExpressionTree {
// in case we get a long.
value = value.toString();
} else {
- throw new MetaException("Invalid key name in filter. " +
+ filterBuilder.setError("Invalid key name in filter. " +
"Use constants from org.apache.hadoop.hive.metastore.api");
+ return;
}
- return generateJDOFilterGeneral(params);
+ generateJDOFilterGeneral(params, filterBuilder);
}
/**
@@ -247,70 +321,73 @@ public class ExpressionTree {
* Currently supported types for value are String and Long.
* The LIKE operator for Longs is unsupported.
*/
- private String generateJDOFilterGeneral(Map<String, Object> params)
- throws MetaException {
+ private void generateJDOFilterGeneral(Map<String, Object> params,
+ FilterBuilder filterBuilder) throws MetaException {
String paramName = PARAM_PREFIX + params.size();
params.put(paramName, value);
- String filter;
if (isReverseOrder) {
if (operator == Operator.LIKE) {
- throw new MetaException(
- "Value should be on the RHS for LIKE operator : " +
+ filterBuilder.setError("Value should be on the RHS for LIKE operator : " +
"Key <" + keyName + ">");
} else {
- filter = paramName + " " + operator.getJdoOp() + " " + keyName;
+ filterBuilder.append(paramName + " " + operator.getJdoOp() + " " + keyName);
}
} else {
if (operator == Operator.LIKE) {
- filter = " " + keyName + "." + operator.getJdoOp() + "(" + paramName + ") ";
+ filterBuilder.append(" " + keyName + "." + operator.getJdoOp() + "(" + paramName + ") ");
} else {
- filter = " " + keyName + " " + operator.getJdoOp() + " " + paramName;
+ filterBuilder.append(" " + keyName + " " + operator.getJdoOp() + " " + paramName);
}
}
- return filter;
}
- private String generateJDOFilterOverPartitions(Table table, Map<String, Object> params)
- throws MetaException {
+ private void generateJDOFilterOverPartitions(Table table, Map<String, Object> params,
+ FilterBuilder filterBuilder) throws MetaException {
int partitionColumnCount = table.getPartitionKeys().size();
- int partitionColumnIndex = getPartColIndexForFilter(table);
+ int partitionColumnIndex = getPartColIndexForFilter(table, filterBuilder);
+ if (filterBuilder.hasError()) return;
+
+ String valueAsString = getFilterPushdownParam(table, partitionColumnIndex, filterBuilder);
+ if (filterBuilder.hasError()) return;
- String valueAsString = getFilterPushdownParam(table, partitionColumnIndex);
String paramName = PARAM_PREFIX + params.size();
params.put(paramName, valueAsString);
boolean isOpEquals = operator == Operator.EQUALS;
if (isOpEquals || operator == Operator.NOTEQUALS || operator == Operator.NOTEQUALS2) {
- return makeFilterForEquals(keyName, valueAsString, paramName, params,
- partitionColumnIndex, partitionColumnCount, isOpEquals);
+ makeFilterForEquals(keyName, valueAsString, paramName, params,
+ partitionColumnIndex, partitionColumnCount, isOpEquals, filterBuilder);
+ return;
}
String keyEqual = FileUtils.escapePathName(keyName) + "=";
- int keyEqualLength = keyEqual.length();
- String valString;
- // partitionname ==> (key=value/)*(key=value)
- if (partitionColumnIndex == (partitionColumnCount - 1)) {
- valString = "partitionName.substring(partitionName.indexOf(\"" + keyEqual + "\")+" + keyEqualLength + ")";
- }
- else {
- valString = "partitionName.substring(partitionName.indexOf(\"" + keyEqual + "\")+" + keyEqualLength + ").substring(0, partitionName.substring(partitionName.indexOf(\"" + keyEqual + "\")+" + keyEqualLength + ").indexOf(\"/\"))";
+ String valString = "partitionName.substring(";
+ String indexOfKeyStr = "";
+ if (partitionColumnIndex != 0) {
+ keyEqual = "/" + keyEqual;
+ indexOfKeyStr = "partitionName.indexOf(\"" + keyEqual + "\") + ";
+ valString += indexOfKeyStr;
+ }
+ valString += keyEqual.length();
+ if (partitionColumnIndex != (partitionColumnCount - 1)) {
+ valString += ", partitionName.indexOf(\"/\", " + indexOfKeyStr + keyEqual.length() + ")";
}
+ valString += ")";
if (operator == Operator.LIKE) {
if (isReverseOrder) {
- //For LIKE, the value should be on the RHS
- throw new MetaException(
+ // For LIKE, the value should be on the RHS.
+ filterBuilder.setError(
"Value should be on the RHS for LIKE operator : Key <" + keyName + ">");
}
- //generate this.values.get(i).matches("abc%")
- return " " + valString + "." + operator.getJdoOp() + "(" + paramName + ") ";
+ // TODO: in all likelihood, this won't actually work. Keep it for backward compat.
+ filterBuilder.append(" " + valString + "." + operator.getJdoOp() + "(" + paramName + ") ");
+ } else {
+ filterBuilder.append(isReverseOrder
+ ? paramName + " " + operator.getJdoOp() + " " + valString
+ : " " + valString + " " + operator.getJdoOp() + " " + paramName);
}
-
- // TODO: support for other ops for numbers to be handled in HIVE-4888.
- return isReverseOrder
- ? paramName + " " + operator.getJdoOp() + " " + valString
- : " " + valString + " " + operator.getJdoOp() + " " + paramName;
}
/**
@@ -339,9 +416,11 @@ public class ExpressionTree {
* Get partition column index in the table partition column list that
* corresponds to the key that is being filtered on by this tree node.
* @param table The table.
+ * @param filterBuilder filter builder used to report error, if any.
* @return The index.
*/
- public int getPartColIndexForFilter(Table table) throws MetaException {
+ public int getPartColIndexForFilter(
+ Table table, FilterBuilder filterBuilder) throws MetaException {
int partitionColumnIndex;
assert (table.getPartitionKeys().size() > 0);
for (partitionColumnIndex = 0; partitionColumnIndex < table.getPartitionKeys().size();
@@ -351,9 +430,10 @@ public class ExpressionTree {
break;
}
}
- if( partitionColumnIndex == table.getPartitionKeys().size() ) {
- throw new MetaException("Specified key <" + keyName +
+ if( partitionColumnIndex == table.getPartitionKeys().size()) {
+ filterBuilder.setError("Specified key <" + keyName +
"> is not a partitioning key for the table");
+ return -1;
}
return partitionColumnIndex;
@@ -365,28 +445,38 @@ public class ExpressionTree {
* In future this may become different for SQL and JDOQL filter pushdown.
* @param table The table.
* @param partColIndex The index of the column to check.
+ * @param filterBuilder filter builder used to report error, if any.
* @return The parameter string.
*/
- public String getFilterPushdownParam(Table table, int partColIndex) throws MetaException {
+ public String getFilterPushdownParam(
+ Table table, int partColIndex, FilterBuilder filterBuilder) throws MetaException {
boolean isIntegralSupported = doesOperatorSupportIntegral(operator);
String colType = table.getPartitionKeys().get(partColIndex).getType();
// Can only support partitions whose types are string, or maybe integers
if (!colType.equals(serdeConstants.STRING_TYPE_NAME)
&& (!isIntegralSupported || !isIntegralType(colType))) {
- throw new MetaException("Filtering is supported only on partition keys of type " +
+ filterBuilder.setError("Filtering is supported only on partition keys of type " +
"string" + (isIntegralSupported ? ", or integral types" : ""));
+ return null;
}
boolean isStringValue = value instanceof String;
if (!isStringValue && (!isIntegralSupported || !(value instanceof Long))) {
- throw new MetaException("Filtering is supported only on partition keys of type " +
+ filterBuilder.setError("Filtering is supported only on partition keys of type " +
"string" + (isIntegralSupported ? ", or integral types" : ""));
+ return null;
}
return isStringValue ? (String) value : Long.toString((Long) value);
}
}
+ public void accept(TreeVisitor treeVisitor) throws MetaException {
+ if (this.root != null) {
+ this.root.accept(treeVisitor);
+ }
+ }
+
/**
* For equals and not-equals, we can make the JDO query much faster by filtering
* based on the partition name. For a condition like ds="2010-10-01", we can see
@@ -400,24 +490,24 @@ public class ExpressionTree {
* Case where the partition key column is at the end of the name. (no
* tailing '/')
*
- * @param keyName name of the partition col e.g. ds
- * @param value
- * @param paramName name of the parameter to use for JDOQL
- * @param params a map from the parameter name to their values
+ * @param keyName name of the partition column e.g. ds.
+ * @param value The value to compare to.
+ * @param paramName name of the parameter to use for JDOQL.
+ * @param params a map from the parameter name to their values.
+ * @param keyPos The index of the requisite partition column in the list of such columns.
+ * @param keyCount Partition column count for the table.
* @param isEq whether the operator is equals, or not-equals.
- * @return
- * @throws MetaException
+ * @param fltr Filter builder used to append the filter, or report errors.
*/
- private static String makeFilterForEquals(String keyName, String value, String paramName,
- Map<String, Object> params, int keyPos, int keyCount, boolean isEq)
- throws MetaException {
+ private static void makeFilterForEquals(String keyName, String value, String paramName,
+ Map<String, Object> params, int keyPos, int keyCount, boolean isEq, FilterBuilder fltr)
+ throws MetaException {
Map<String, String> partKeyToVal = new HashMap<String, String>();
partKeyToVal.put(keyName, value);
// If a partition has multiple partition keys, we make the assumption that
// makePartName with one key will return a substring of the name made
// with both all the keys.
String escapedNameFragment = Warehouse.makePartName(partKeyToVal, false);
- StringBuilder fltr = new StringBuilder();
if (keyCount == 1) {
// Case where this is no other partition columns
params.put(paramName, escapedNameFragment);
@@ -427,13 +517,13 @@ public class ExpressionTree {
// be a leading '/' but no trailing '/'
params.put(paramName, "/" + escapedNameFragment);
fltr.append(isEq ? "" : "!").append("partitionName.endsWith(")
- .append(paramName).append(')');
+ .append(paramName).append(")");
} else if (keyPos == 0) {
- // Case where the parttion column is at the beginning of the name. There will
+ // Case where the partition column is at the beginning of the name. There will
// be a trailing '/' but no leading '/'
params.put(paramName, escapedNameFragment + "/");
fltr.append(isEq ? "" : "!").append("partitionName.startsWith(")
- .append(paramName).append(')');
+ .append(paramName).append(")");
} else {
// Case where the partition column is in the middle of the name. There will
// be a leading '/' and an trailing '/'
@@ -441,7 +531,6 @@ public class ExpressionTree {
fltr.append("partitionName.indexOf(").append(paramName).append(")")
.append(isEq ? ">= 0" : "< 0");
}
- return fltr.toString();
}
/**
@@ -489,19 +578,19 @@ public class ExpressionTree {
* @param params the input map which is updated with the
* the parameterized values. Keys are the parameter names and values
* are the parameter values
- * @return the string representation of the expression tree
- * @throws MetaException
+ * @param filterBuilder the filter builder to append to.
*/
- public String generateJDOFilter(Table table,
- Map<String, Object> params) throws MetaException {
- if( root == null ) {
- return "";
+ public void generateJDOFilterFragment(Table table, Map<String, Object> params,
+ FilterBuilder filterBuilder) throws MetaException {
+ if (root == null) {
+ return;
}
- return root.generateJDOFilter(table, params);
+ filterBuilder.append(" && ( ");
+ root.generateJDOFilter(table, params, filterBuilder);
+ filterBuilder.append(" )");
}
-
/** Case insensitive ANTLR string stream */
public static class ANTLRNoCaseStringStream extends ANTLRStringStream {
public ANTLRNoCaseStringStream (String input) {
Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java?rev=1526106&r1=1526105&r2=1526106&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java Wed Sep 25 04:57:46 2013
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.metastore
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -50,6 +51,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.model.MRoleMap;
import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege;
import org.apache.hadoop.hive.metastore.model.MTablePrivilege;
+import org.apache.thrift.TException;
/**
* A wrapper around {@link org.apache.hadoop.hive.metastore.ObjectStore}
@@ -303,6 +305,13 @@ public class DummyRawStoreControlledComm
}
@Override
+ public boolean getPartitionsByExpr(String dbName, String tblName, byte[] expr,
+ String defaultPartitionName, short maxParts, Set<Partition> result) throws TException {
+ return objectStore.getPartitionsByExpr(
+ dbName, tblName, expr, defaultPartitionName, maxParts, result);
+ }
+
+ @Override
public Table markPartitionForEvent(String dbName, String tblName,
Map<String, String> partVals, PartitionEventType evtType)
throws MetaException, UnknownTableException, InvalidPartitionException,
Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java?rev=1526106&r1=1526105&r2=1526106&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java Wed Sep 25 04:57:46 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.metastore
import java.util.List;
import java.util.Map;
+import java.util.Set;
import junit.framework.Assert;
@@ -51,6 +52,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.model.MRoleMap;
import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege;
import org.apache.hadoop.hive.metastore.model.MTablePrivilege;
+import org.apache.thrift.TException;
/**
*
@@ -322,6 +324,12 @@ public class DummyRawStoreForJdoConnecti
}
@Override
+ public boolean getPartitionsByExpr(String dbName, String tblName, byte[] expr,
+ String defaultPartitionName, short maxParts, Set<Partition> result) throws TException {
+ return false;
+ }
+
+ @Override
public Table markPartitionForEvent(String dbName, String tblName, Map<String, String> partVals,
PartitionEventType evtType) throws MetaException, UnknownTableException,
InvalidPartitionException, UnknownPartitionException {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1526106&r1=1526105&r2=1526106&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Wed Sep 25 04:57:46 2013
@@ -176,6 +176,7 @@ public final class FunctionRegistry {
*/
static Map<String, FunctionInfo> mFunctions = Collections.synchronizedMap(new LinkedHashMap<String, FunctionInfo>());
+ static Set<Class<?>> nativeUdfs = Collections.synchronizedSet(new HashSet<Class<?>>());
/*
* PTF variables
* */
@@ -486,6 +487,7 @@ public final class FunctionRegistry {
FunctionInfo fI = new FunctionInfo(isNative, displayName,
new GenericUDFBridge(displayName, isOperator, UDFClass.getName()));
mFunctions.put(functionName.toLowerCase(), fI);
+ registerNativeStatus(fI);
} else {
throw new RuntimeException("Registering UDF Class " + UDFClass
+ " which does not extend " + UDF.class);
@@ -508,6 +510,7 @@ public final class FunctionRegistry {
FunctionInfo fI = new FunctionInfo(isNative, functionName,
(GenericUDF) ReflectionUtils.newInstance(genericUDFClass, null));
mFunctions.put(functionName.toLowerCase(), fI);
+ registerNativeStatus(fI);
} else {
throw new RuntimeException("Registering GenericUDF Class "
+ genericUDFClass + " which does not extend " + GenericUDF.class);
@@ -530,6 +533,7 @@ public final class FunctionRegistry {
FunctionInfo fI = new FunctionInfo(isNative, functionName,
(GenericUDTF) ReflectionUtils.newInstance(genericUDTFClass, null));
mFunctions.put(functionName.toLowerCase(), fI);
+ registerNativeStatus(fI);
} else {
throw new RuntimeException("Registering GenericUDTF Class "
+ genericUDTFClass + " which does not extend " + GenericUDTF.class);
@@ -955,8 +959,9 @@ public final class FunctionRegistry {
public static void registerGenericUDAF(boolean isNative, String functionName,
GenericUDAFResolver genericUDAFResolver) {
- mFunctions.put(functionName.toLowerCase(), new FunctionInfo(isNative,
- functionName.toLowerCase(), genericUDAFResolver));
+ FunctionInfo fi = new FunctionInfo(isNative, functionName.toLowerCase(), genericUDAFResolver);
+ mFunctions.put(functionName.toLowerCase(), fi);
+ registerNativeStatus(fi);
}
public static void registerTemporaryUDAF(String functionName,
@@ -970,9 +975,11 @@ public final class FunctionRegistry {
public static void registerUDAF(boolean isNative, String functionName,
Class<? extends UDAF> udafClass) {
- mFunctions.put(functionName.toLowerCase(), new FunctionInfo(isNative,
+ FunctionInfo fi = new FunctionInfo(isNative,
functionName.toLowerCase(), new GenericUDAFBridge(
- (UDAF) ReflectionUtils.newInstance(udafClass, null))));
+ (UDAF) ReflectionUtils.newInstance(udafClass, null)));
+ mFunctions.put(functionName.toLowerCase(), fi);
+ registerNativeStatus(fi);
}
public static void unregisterTemporaryUDF(String functionName) throws HiveException {
@@ -1560,6 +1567,7 @@ public final class FunctionRegistry {
FunctionInfo fI = new FunctionInfo(false, macroName,
new GenericUDFMacro(macroName, body, colNames, colTypes));
mFunctions.put(macroName.toLowerCase(), fI);
+ registerNativeStatus(fI);
}
/**
@@ -1717,6 +1725,7 @@ public final class FunctionRegistry {
{
FunctionInfo tInfo = new FunctionInfo(name, tFnCls);
mFunctions.put(name.toLowerCase(), tInfo);
+ registerNativeStatus(tInfo);
}
/**
@@ -1738,4 +1747,21 @@ public final class FunctionRegistry {
}
return false;
}
+
+ /**
+ * @param fnExpr Function expression.
+ * @return True iff the fnExpr represents a hive built-in function.
+ */
+ public static boolean isNativeFuncExpr(ExprNodeGenericFuncDesc fnExpr) {
+ Class<?> udfClass = getUDFClassFromExprDesc(fnExpr);
+ if (udfClass == null) {
+ udfClass = getGenericUDFClassFromExprDesc(fnExpr);
+ }
+ return nativeUdfs.contains(udfClass);
+ }
+
+ private static void registerNativeStatus(FunctionInfo fi) {
+ if (!fi.isNative()) return;
+ nativeUdfs.add(fi.getFunctionClass());
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1526106&r1=1526105&r2=1526106&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Sep 25 04:57:46 2013
@@ -129,10 +129,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
-import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.MapWork;
@@ -148,16 +145,10 @@ import org.apache.hadoop.hive.ql.plan.ap
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.stats.StatsFactory;
import org.apache.hadoop.hive.ql.stats.StatsPublisher;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.Serializer;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
@@ -586,6 +577,31 @@ public final class Utilities {
return null;
}
+ /**
+ * Serializes expression via Kryo.
+ * @param expr Expression.
+ * @return Bytes.
+ */
+ public static byte[] serializeExpressionToKryo(ExprNodeDesc expr) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ Output output = new Output(baos);
+ runtimeSerializationKryo.get().writeClassAndObject(output, expr);
+ output.close();
+ return baos.toByteArray();
+ }
+
+ /**
+ * Deserializes expression from Kryo.
+ * @param bytes Bytes containing the expression.
+ * @return Expression; null if deserialization succeeded, but the result type is incorrect.
+ */
+ public static ExprNodeDesc deserializeExpressionFromKryo(byte[] bytes) {
+ Input inp = new Input(new ByteArrayInputStream(bytes));
+ Object o = runtimeSerializationKryo.get().readClassAndObject(inp);
+ inp.close();
+ return (o instanceof ExprNodeDesc) ? (ExprNodeDesc)o : null;
+ }
+
public static String serializeExpression(ExprNodeDesc expr) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
XMLEncoder encoder = new XMLEncoder(baos);
@@ -652,7 +668,6 @@ public final class Utilities {
public void write(Kryo kryo, Output output, java.sql.Date sqlDate) {
output.writeLong(sqlDate.getTime());
}
-
}
private static class CommonTokenSerializer extends com.esotericsoftware.kryo.Serializer<CommonToken> {
@@ -2367,126 +2382,6 @@ public final class Utilities {
}
/**
- * Check if a function can be pushed down to JDO.
- * Now only {compares, AND, OR} are supported.
- * @param func a generic function.
- * @return true if this function can be pushed down to JDO filter.
- */
- private static boolean supportedJDOFuncs(GenericUDF func) {
- // TODO: we might also want to add "not" and "between" here in future.
- // TODO: change to GenericUDFBaseCompare once DN is upgraded
- // (see HIVE-2609 - in DN 2.0, substrings do not work in MySQL).
- return func instanceof GenericUDFOPEqual ||
- func instanceof GenericUDFOPNotEqual ||
- func instanceof GenericUDFOPAnd ||
- func instanceof GenericUDFOPOr;
- }
-
- /**
- * Check if a function can be pushed down to JDO for integral types.
- * Only {=, !=} are supported. lt/gt/etc. to be dealt with in HIVE-4888.
- * @param func a generic function.
- * @return true iff this function can be pushed down to JDO filter for integral types.
- */
- private static boolean doesJDOFuncSupportIntegral(GenericUDF func) {
- // AND, OR etc. don't need to be specified here.
- return func instanceof GenericUDFOPEqual ||
- func instanceof GenericUDFOPNotEqual;
- }
-
- /**
- * @param type type
- * @param constant The constant, if any.
- * @return true iff type is an integral type.
- */
- private static boolean isIntegralType(String type) {
- return type.equals(serdeConstants.TINYINT_TYPE_NAME) ||
- type.equals(serdeConstants.SMALLINT_TYPE_NAME) ||
- type.equals(serdeConstants.INT_TYPE_NAME) ||
- type.equals(serdeConstants.BIGINT_TYPE_NAME);
- }
-
- /**
- * Check if the partition pruning expression can be pushed down to JDO filtering.
- * The partition expression contains only partition columns.
- * The criteria that an expression can be pushed down are that:
- * 1) the expression only contains function specified in supportedJDOFuncs().
- * Now only {=, AND, OR} can be pushed down.
- * 2) the partition column type and the constant type have to be String. This is
- * restriction by the current JDO filtering implementation.
- * @param tab The table that contains the partition columns.
- * @param expr the partition pruning expression
- * @param parent parent UDF of expr if parent exists and contains a UDF; otherwise null.
- * @return null if the partition pruning expression can be pushed down to JDO filtering.
- */
- public static String checkJDOPushDown(
- Table tab, ExprNodeDesc expr, GenericUDF parent) {
- boolean isConst = expr instanceof ExprNodeConstantDesc;
- boolean isCol = !isConst && (expr instanceof ExprNodeColumnDesc);
- boolean isIntegralSupported = (parent != null) && (isConst || isCol)
- && doesJDOFuncSupportIntegral(parent);
-
- // JDO filter now only support String typed literals, as well as integers
- // for some operators; see Filter.g and ExpressionTree.java.
- if (isConst) {
- Object value = ((ExprNodeConstantDesc)expr).getValue();
- if (value instanceof String) {
- return null;
- }
- if (isIntegralSupported && isIntegralType(expr.getTypeInfo().getTypeName())) {
- return null;
- }
- return "Constant " + value + " is not string "
- + (isIntegralSupported ? "or integral ": "") + "type: " + expr.getTypeInfo().getTypeName();
- } else if (isCol) {
- TypeInfo type = expr.getTypeInfo();
- if (type.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)
- || (isIntegralSupported && isIntegralType(type.getTypeName()))) {
- String colName = ((ExprNodeColumnDesc)expr).getColumn();
- for (FieldSchema fs: tab.getPartCols()) {
- if (fs.getName().equals(colName)) {
- if (fs.getType().equals(serdeConstants.STRING_TYPE_NAME)
- || (isIntegralSupported && isIntegralType(fs.getType()))) {
- return null;
- }
- return "Partition column " + fs.getName() + " is not string "
- + (isIntegralSupported ? "or integral ": "") + "type: " + fs.getType();
- }
- }
- assert(false); // cannot find the partition column!
- } else {
- return "Column " + expr.getExprString() + " is not string "
- + (isIntegralSupported ? "or integral ": "") + "type: " + type.getTypeName();
- }
- } else if (expr instanceof ExprNodeGenericFuncDesc) {
- ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) expr;
- GenericUDF func = funcDesc.getGenericUDF();
- if (!supportedJDOFuncs(func)) {
- return "Expression " + expr.getExprString() + " cannot be evaluated";
- }
- boolean allChildrenConstant = true;
- List<ExprNodeDesc> children = funcDesc.getChildExprs();
- for (ExprNodeDesc child: children) {
- if (!(child instanceof ExprNodeConstantDesc)) {
- allChildrenConstant = false;
- }
- String message = checkJDOPushDown(tab, child, func);
- if (message != null) {
- return message;
- }
- }
-
- // If all the children of the expression are constants then JDO cannot parse the expression
- // see Filter.g
- if (allChildrenConstant) {
- return "Expression " + expr.getExprString() + " has only constants as children.";
- }
- return null;
- }
- return "Expression " + expr.getExprString() + " cannot be evaluated";
- }
-
- /**
* The check here is kind of not clean. It first use a for loop to go through
* all input formats, and choose the ones that extend ReworkMapredInputFormat
* to a set. And finally go through the ReworkMapredInputFormat set, and call
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1526106&r1=1526105&r2=1526106&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Wed Sep 25 04:57:46 2013
@@ -49,6 +49,7 @@ import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaException;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
@@ -80,6 +81,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.index.HiveIndexHandler;
import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.session.CreateTableAutomaticGrant;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
@@ -1899,13 +1901,43 @@ private void constructOneLBLocationMap(F
List<org.apache.hadoop.hive.metastore.api.Partition> tParts = getMSC().listPartitionsByFilter(
tbl.getDbName(), tbl.getTableName(), filter, (short)-1);
- List<Partition> results = new ArrayList<Partition>(tParts.size());
+ return convertFromMetastore(tbl, tParts, null);
+ }
- for (org.apache.hadoop.hive.metastore.api.Partition tPart: tParts) {
- Partition part = new Partition(tbl, tPart);
- results.add(part);
+ private static List<Partition> convertFromMetastore(Table tbl,
+ List<org.apache.hadoop.hive.metastore.api.Partition> src,
+ List<Partition> dest) throws HiveException {
+ if (src == null) {
+ return dest;
+ }
+ if (dest == null) {
+ dest = new ArrayList<Partition>(src.size());
}
- return results;
+ for (org.apache.hadoop.hive.metastore.api.Partition tPart : src) {
+ dest.add(new Partition(tbl, tPart));
+ }
+ return dest;
+ }
+
+ /**
+ * Get a list of Partitions by expr.
+ * @param tbl The table containing the partitions.
+ * @param expr A serialized expression for partition predicates.
+ * @param conf Hive config.
+ * @param result the resulting list of partitions
+ * @return whether the resulting list contains partitions which may or may not match the expr
+ */
+ public boolean getPartitionsByExpr(Table tbl, ExprNodeDesc expr, HiveConf conf,
+ List<Partition> result) throws HiveException, TException {
+ assert result != null;
+ byte[] exprBytes = Utilities.serializeExpressionToKryo(expr);
+ String defaultPartitionName = HiveConf.getVar(conf, ConfVars.DEFAULTPARTITIONNAME);
+ List<org.apache.hadoop.hive.metastore.api.Partition> msParts =
+ new ArrayList<org.apache.hadoop.hive.metastore.api.Partition>();
+ boolean hasUnknownParts = getMSC().listPartitionsByExpr(tbl.getDbName(),
+ tbl.getTableName(), exprBytes, defaultPartitionName, (short)-1, msParts);
+ convertFromMetastore(tbl, msParts, result);
+ return hasUnknownParts;
}
public void validatePartitionNameCharacters(List<String> partVals) throws HiveException {
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java?rev=1526106&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java Wed Sep 25 04:57:46 2013
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.ppr;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+
+/**
+ * The basic implementation of PartitionExpressionProxy that uses ql package classes.
+ */
+public class PartitionExpressionForMetastore implements PartitionExpressionProxy {
+ private static final Log LOG = LogFactory.getLog(PartitionExpressionForMetastore.class);
+
+ @Override
+ public String convertExprToFilter(byte[] exprBytes) throws MetaException {
+ return deserializeExpr(exprBytes).getExprString();
+ }
+
+ @Override
+ public boolean filterPartitionsByExpr(List<String> columnNames, byte[] exprBytes,
+ String defaultPartitionName, List<String> partitionNames) throws MetaException {
+ ExprNodeDesc expr = deserializeExpr(exprBytes);
+ try {
+ long startTime = System.nanoTime(), len = partitionNames.size();
+ boolean result = PartitionPruner.prunePartitionNames(
+ columnNames, expr, defaultPartitionName, partitionNames);
+ double timeMs = (System.nanoTime() - startTime) / 1000000.0;
+ LOG.debug("Pruning " + len + " partition names took " + timeMs + "ms");
+ return result;
+ } catch (HiveException ex) {
+ LOG.error("Failed to apply the expression", ex);
+ throw new MetaException(ex.getMessage());
+ }
+ }
+
+ private ExprNodeDesc deserializeExpr(byte[] exprBytes) throws MetaException {
+ ExprNodeDesc expr = null;
+ try {
+ expr = Utilities.deserializeExpressionFromKryo(exprBytes);
+ } catch (Exception ex) {
+ LOG.error("Failed to deserialize the expression", ex);
+ throw new MetaException(ex.getMessage());
+ }
+ if (expr == null) {
+ throw new MetaException("Failed to deserialize expression - ExprNodeDesc not present");
+ }
+ return expr;
+ }
+}