You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/11/18 01:48:46 UTC
svn commit: r1640263 [7/12] - in /hive/branches/spark: ./
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/ accu...
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java Tue Nov 18 00:48:40 2014
@@ -57,7 +57,9 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -420,9 +422,10 @@ public class PartitionPruner implements
String defaultPartitionName = conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME);
List<String> partCols = extractPartColNames(tab);
+ List<PrimitiveTypeInfo> partColTypeInfos = extractPartColTypes(tab);
boolean hasUnknownPartitions = prunePartitionNames(
- partCols, prunerExpr, defaultPartitionName, partNames);
+ partCols, partColTypeInfos, prunerExpr, defaultPartitionName, partNames);
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PRUNE_LISTING);
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING);
@@ -442,19 +445,30 @@ public class PartitionPruner implements
return partCols;
}
+ private static List<PrimitiveTypeInfo> extractPartColTypes(Table tab) {
+ List<FieldSchema> pCols = tab.getPartCols();
+ List<PrimitiveTypeInfo> partColTypeInfos = new ArrayList<PrimitiveTypeInfo>(pCols.size());
+ for (FieldSchema pCol : pCols) {
+ partColTypeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(pCol.getType()));
+ }
+ return partColTypeInfos;
+ }
+
/**
* Prunes partition names to see if they match the prune expression.
- * @param columnNames name of partition columns
+ * @param partColumnNames name of partition columns
+ * @param partColumnTypeInfos types of partition columns
* @param prunerExpr The expression to match.
* @param defaultPartitionName name of default partition
* @param partNames Partition names to filter. The list is modified in place.
* @return Whether the list has any partitions for which the expression may or may not match.
*/
- public static boolean prunePartitionNames(List<String> columnNames, ExprNodeGenericFuncDesc prunerExpr,
+ public static boolean prunePartitionNames(List<String> partColumnNames,
+ List<PrimitiveTypeInfo> partColumnTypeInfos, ExprNodeGenericFuncDesc prunerExpr,
String defaultPartitionName, List<String> partNames) throws HiveException, MetaException {
// Prepare the expression to filter on the columns.
ObjectPair<PrimitiveObjectInspector, ExprNodeEvaluator> handle =
- PartExprEvalUtils.prepareExpr(prunerExpr, columnNames);
+ PartExprEvalUtils.prepareExpr(prunerExpr, partColumnNames, partColumnTypeInfos);
// Filter the name list. Removing elements one by one can be slow on e.g. ArrayList,
// so let's create a new list and copy it if we don't have a linked list
@@ -462,8 +476,8 @@ public class PartitionPruner implements
List<String> partNamesSeq = inPlace ? partNames : new LinkedList<String>(partNames);
// Array for the values to pass to evaluator.
- ArrayList<String> values = new ArrayList<String>(columnNames.size());
- for (int i = 0; i < columnNames.size(); ++i) {
+ ArrayList<String> values = new ArrayList<String>(partColumnNames.size());
+ for (int i = 0; i < partColumnNames.size(); ++i) {
values.add(null);
}
@@ -473,8 +487,17 @@ public class PartitionPruner implements
String partName = partIter.next();
Warehouse.makeValsFromName(partName, values);
+ ArrayList<Object> convertedValues = new ArrayList<Object>(values.size());
+ for(int i=0; i<values.size(); i++) {
+ Object o = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(partColumnTypeInfos.get(i)))
+ .convert(values.get(i));
+ convertedValues.add(o);
+ }
+
// Evaluate the expression tree.
- Boolean isNeeded = (Boolean)PartExprEvalUtils.evaluateExprOnPart(handle, values);
+ Boolean isNeeded = (Boolean)PartExprEvalUtils.evaluateExprOnPart(handle, convertedValues);
boolean isUnknown = (isNeeded == null);
if (!isUnknown && !isNeeded) {
partIter.remove();
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Tue Nov 18 00:48:40 2014
@@ -22,6 +22,8 @@ import static org.apache.hadoop.hive.ql.
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DATABASEPROPERTIES;
import java.io.Serializable;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
@@ -229,7 +231,7 @@ public class DDLSemanticAnalyzer extends
reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_ORIGINAL));
reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_ARCHIVED));
reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_EXTRACTED));
- hiveAuthorizationTaskFactory = new HiveAuthorizationTaskFactoryImpl(conf, db);
+ hiveAuthorizationTaskFactory = createAuthorizationTaskFactory(conf, db);
}
@Override
@@ -3419,4 +3421,30 @@ public class DDLSemanticAnalyzer extends
throw new SemanticException(e);
}
}
+
+ private HiveAuthorizationTaskFactory createAuthorizationTaskFactory(HiveConf conf, Hive db) {
+ Class<? extends HiveAuthorizationTaskFactory> authProviderClass = conf.
+ getClass(HiveConf.ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY.varname,
+ HiveAuthorizationTaskFactoryImpl.class,
+ HiveAuthorizationTaskFactory.class);
+ String msg = "Unable to create instance of " + authProviderClass.getName() + ": ";
+ try {
+ Constructor<? extends HiveAuthorizationTaskFactory> constructor =
+ authProviderClass.getConstructor(HiveConf.class, Hive.class);
+ return constructor.newInstance(conf, db);
+ } catch (NoSuchMethodException e) {
+ throw new IllegalStateException(msg + e.getMessage(), e);
+ } catch (SecurityException e) {
+ throw new IllegalStateException(msg + e.getMessage(), e);
+ } catch (InstantiationException e) {
+ throw new IllegalStateException(msg + e.getMessage(), e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalStateException(msg + e.getMessage(), e);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalStateException(msg + e.getMessage(), e);
+ } catch (InvocationTargetException e) {
+ throw new IllegalStateException(msg + e.getMessage(), e);
+ }
+ }
+
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Tue Nov 18 00:48:40 2014
@@ -283,6 +283,8 @@ KW_ROLE: 'ROLE';
KW_ROLES: 'ROLES';
KW_INNER: 'INNER';
KW_EXCHANGE: 'EXCHANGE';
+KW_URI: 'URI';
+KW_SERVER : 'SERVER';
KW_ADMIN: 'ADMIN';
KW_OWNER: 'OWNER';
KW_PRINCIPALS: 'PRINCIPALS';
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Tue Nov 18 00:48:40 2014
@@ -109,6 +109,7 @@ TOK_DATE;
TOK_DATELITERAL;
TOK_DATETIME;
TOK_TIMESTAMP;
+TOK_TIMESTAMPLITERAL;
TOK_STRING;
TOK_CHAR;
TOK_VARCHAR;
@@ -340,6 +341,8 @@ TOK_VIRTUAL_TABLE;
TOK_VIRTUAL_TABREF;
TOK_ANONYMOUS;
TOK_COL_NAME;
+TOK_URI_TYPE;
+TOK_SERVER_TYPE;
}
@@ -1485,11 +1488,15 @@ privilegeObject
privObject
: (KW_DATABASE|KW_SCHEMA) identifier -> ^(TOK_DB_TYPE identifier)
| KW_TABLE? tableName partitionSpec? -> ^(TOK_TABLE_TYPE tableName partitionSpec?)
+ | KW_URI (path=StringLiteral) -> ^(TOK_URI_TYPE $path)
+ | KW_SERVER identifier -> ^(TOK_SERVER_TYPE identifier)
;
privObjectCols
: (KW_DATABASE|KW_SCHEMA) identifier -> ^(TOK_DB_TYPE identifier)
| KW_TABLE? tableName (LPAREN cols=columnNameList RPAREN)? partitionSpec? -> ^(TOK_TABLE_TYPE tableName $cols? partitionSpec?)
+ | KW_URI (path=StringLiteral) -> ^(TOK_URI_TYPE $path)
+ | KW_SERVER identifier -> ^(TOK_SERVER_TYPE identifier)
;
privilegeList
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Tue Nov 18 00:48:40 2014
@@ -218,6 +218,7 @@ constant
:
Number
| dateLiteral
+ | timestampLiteral
| StringLiteral
| stringLiteralSequence
| BigintLiteral
@@ -250,6 +251,14 @@ dateLiteral
}
;
+timestampLiteral
+ :
+ KW_TIMESTAMP StringLiteral ->
+ {
+ adaptor.create(TOK_TIMESTAMPLITERAL, $StringLiteral.text)
+ }
+ ;
+
expression
@init { gParent.pushMsg("expression specification", state); }
@after { gParent.popMsg(state); }
@@ -260,7 +269,6 @@ expression
atomExpression
:
KW_NULL -> TOK_NULL
- | dateLiteral
| constant
| castExpression
| caseExpression
@@ -545,5 +553,5 @@ principalIdentifier
nonReserved
:
- KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION |
KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_AN
ALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES
+ KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION |
KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_AN
ALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES | KW_URI | KW_SERVER
;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Nov 18 00:48:40 2014
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryProperties;
import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
@@ -318,9 +319,6 @@ public class SemanticAnalyzer extends Ba
private static final String VALUES_TMP_TABLE_NAME_PREFIX = "Values__Tmp__Table__";
- @VisibleForTesting
- static final String ACID_TABLE_PROPERTY = "transactional";
-
private HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner;
private HashMap<TableScanOperator, PrunedPartitionList> opToPartList;
private HashMap<String, Operator<? extends OperatorDesc>> topOps;
@@ -2764,11 +2762,14 @@ public class SemanticAnalyzer extends Ba
// TODO: make aliases unique, otherwise needless rewriting takes place
private Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel,
ArrayList<ExprNodeDesc> col_list, HashSet<ColumnInfo> excludeCols, RowResolver input,
- Integer pos, RowResolver output, List<String> aliases, boolean ensureUniqueCols)
- throws SemanticException {
+ RowResolver colSrcRR, Integer pos, RowResolver output, List<String> aliases,
+ boolean ensureUniqueCols) throws SemanticException {
+ if (colSrcRR == null) {
+ colSrcRR = input;
+ }
// The table alias should exist
- if (tabAlias != null && !input.hasTableAlias(tabAlias)) {
+ if (tabAlias != null && !colSrcRR.hasTableAlias(tabAlias)) {
throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(sel));
}
@@ -2797,7 +2798,7 @@ public class SemanticAnalyzer extends Ba
// For expr "*", aliases should be iterated in the order they are specified
// in the query.
for (String alias : aliases) {
- HashMap<String, ColumnInfo> fMap = input.getFieldMap(alias);
+ HashMap<String, ColumnInfo> fMap = colSrcRR.getFieldMap(alias);
if (fMap == null) {
continue;
}
@@ -2808,8 +2809,11 @@ public class SemanticAnalyzer extends Ba
if (excludeCols != null && excludeCols.contains(colInfo)) {
continue; // This was added during plan generation.
}
+ // First, look up the column from the source against which * is to be resolved.
+ // We'd later translated this into the column from proper input, if it's valid.
+ // TODO: excludeCols may be possible to remove using the same technique.
String name = colInfo.getInternalName();
- String[] tmp = input.reverseLookup(name);
+ String[] tmp = colSrcRR.reverseLookup(name);
// Skip the colinfos which are not for this particular alias
if (tabAlias != null && !tmp[0].equalsIgnoreCase(tabAlias)) {
@@ -2825,6 +2829,27 @@ public class SemanticAnalyzer extends Ba
continue;
}
+ // If input (GBY) is different than the source of columns, find the same column in input.
+ // TODO: This is fraught with peril.
+ if (input != colSrcRR) {
+ colInfo = input.get(tabAlias, tmp[1]);
+ if (colInfo == null) {
+ LOG.error("Cannot find colInfo for " + tabAlias + "." + tmp[1]
+ + ", derived from [" + colSrcRR + "], in [" + input + "]");
+ throw new SemanticException(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY, tmp[1]);
+ }
+ String oldCol = null;
+ if (LOG.isDebugEnabled()) {
+ oldCol = name + " => " + (tmp == null ? "null" : (tmp[0] + "." + tmp[1]));
+ }
+ name = colInfo.getInternalName();
+ tmp = input.reverseLookup(name);
+ if (LOG.isDebugEnabled()) {
+ String newCol = name + " => " + (tmp == null ? "null" : (tmp[0] + "." + tmp[1]));
+ LOG.debug("Translated [" + oldCol + "] to [" + newCol + "]");
+ }
+ }
+
ColumnInfo oColInfo = inputColsProcessed.get(colInfo);
if (oColInfo == null) {
ExprNodeColumnDesc expr = new ExprNodeColumnDesc(colInfo.getType(),
@@ -3423,11 +3448,10 @@ public class SemanticAnalyzer extends Ba
}
- private Operator<?> genSelectPlan(String dest, QB qb, Operator<?> input)
- throws SemanticException {
+ private Operator<?> genSelectPlan(String dest, QB qb, Operator<?> input,
+ Operator<?> inputForSelectStar) throws SemanticException {
ASTNode selExprList = qb.getParseInfo().getSelForClause(dest);
-
- Operator<?> op = genSelectPlan(selExprList, qb, input, false);
+ Operator<?> op = genSelectPlan(selExprList, qb, input, inputForSelectStar, false);
if (LOG.isDebugEnabled()) {
LOG.debug("Created Select Plan for clause: " + dest);
@@ -3437,8 +3461,8 @@ public class SemanticAnalyzer extends Ba
}
@SuppressWarnings("nls")
- private Operator<?> genSelectPlan(ASTNode selExprList, QB qb,
- Operator<?> input, boolean outerLV) throws SemanticException {
+ private Operator<?> genSelectPlan(ASTNode selExprList, QB qb, Operator<?> input,
+ Operator<?> inputForSelectStar, boolean outerLV) throws SemanticException {
if (LOG.isDebugEnabled()) {
LOG.debug("tree: " + selExprList.toStringTree());
@@ -3449,6 +3473,10 @@ public class SemanticAnalyzer extends Ba
ASTNode trfm = null;
Integer pos = Integer.valueOf(0);
RowResolver inputRR = opParseCtx.get(input).getRowResolver();
+ RowResolver starRR = null;
+ if (inputForSelectStar != null && inputForSelectStar != input) {
+ starRR = opParseCtx.get(inputForSelectStar).getRowResolver();
+ }
// SELECT * or SELECT TRANSFORM(*)
boolean selectStar = false;
int posn = 0;
@@ -3494,7 +3522,7 @@ public class SemanticAnalyzer extends Ba
}
if (isUDTF && (selectStar = udtfExprType == HiveParser.TOK_FUNCTIONSTAR)) {
genColListRegex(".*", null, (ASTNode) udtfExpr.getChild(0),
- col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false);
+ col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false);
}
}
@@ -3547,7 +3575,7 @@ public class SemanticAnalyzer extends Ba
}
if (LOG.isDebugEnabled()) {
- LOG.debug("genSelectPlan: input = " + inputRR.toString());
+ LOG.debug("genSelectPlan: input = " + inputRR + " starRr = " + starRR);
}
// For UDTF's, skip the function name to get the expressions
@@ -3616,7 +3644,7 @@ public class SemanticAnalyzer extends Ba
if (expr.getType() == HiveParser.TOK_ALLCOLREF) {
pos = genColListRegex(".*", expr.getChildCount() == 0 ? null
: getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(),
- expr, col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false);
+ expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false);
selectStar = true;
} else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause
&& !inputRR.getIsExprResolver()
@@ -3625,7 +3653,7 @@ public class SemanticAnalyzer extends Ba
// This can only happen without AS clause
// We don't allow this for ExprResolver - the Group By case
pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()),
- null, expr, col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false);
+ null, expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false);
} else if (expr.getType() == HiveParser.DOT
&& expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL
&& inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0)
@@ -3637,7 +3665,7 @@ public class SemanticAnalyzer extends Ba
// We don't allow this for ExprResolver - the Group By case
pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()),
unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()),
- expr, col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false);
+ expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false);
} else {
// Case when this is an expression
TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR);
@@ -5165,7 +5193,8 @@ public class SemanticAnalyzer extends Ba
Operator groupByOperatorInfo = genGroupByPlanGroupByOperator(parseInfo,
dest, curr, reduceSinkOperatorInfo, GroupByDesc.Mode.COMPLETE, null);
- curr = genPostGroupByBodyPlan(groupByOperatorInfo, dest, qb, aliasToOpInfo);
+ // TODO: should we pass curr instead of null?
+ curr = genPostGroupByBodyPlan(groupByOperatorInfo, dest, qb, aliasToOpInfo, null);
}
return curr;
@@ -8782,7 +8811,7 @@ public class SemanticAnalyzer extends Ba
for (String dest : ks) {
curr = input;
curr = genGroupByPlan2MRMultiGroupBy(dest, qb, curr);
- curr = genSelectPlan(dest, qb, curr);
+ curr = genSelectPlan(dest, qb, curr, null); // TODO: we may need to pass "input" here instead of null
Integer limit = qbp.getDestLimit(dest);
if (limit != null) {
curr = genLimitMapRedPlan(dest, qb, curr, limit.intValue(), true);
@@ -8839,6 +8868,8 @@ public class SemanticAnalyzer extends Ba
ASTNode whereExpr = qb.getParseInfo().getWhrForClause(dest);
curr = genFilterPlan((ASTNode) whereExpr.getChild(0), qb, curr, aliasToOpInfo, false);
}
+ // Preserve operator before the GBY - we'll use it to resolve '*'
+ Operator<?> gbySource = curr;
if (qbp.getAggregationExprsForClause(dest).size() != 0
|| getGroupByForClause(qbp, dest).size() > 0) {
@@ -8863,8 +8894,12 @@ public class SemanticAnalyzer extends Ba
curr = genGroupByPlan1MR(dest, qb, curr);
}
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("RR before GB " + opParseCtx.get(gbySource).getRowResolver()
+ + " after GB " + opParseCtx.get(curr).getRowResolver());
+ }
- curr = genPostGroupByBodyPlan(curr, dest, qb, aliasToOpInfo);
+ curr = genPostGroupByBodyPlan(curr, dest, qb, aliasToOpInfo, gbySource);
}
} else {
curr = genGroupByPlan1ReduceMultiGBY(commonGroupByDestGroup, qb, input, aliasToOpInfo);
@@ -8891,7 +8926,7 @@ public class SemanticAnalyzer extends Ba
}
private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb,
- Map<String, Operator> aliasToOpInfo)
+ Map<String, Operator> aliasToOpInfo, Operator gbySource)
throws SemanticException {
QBParseInfo qbp = qb.getParseInfo();
@@ -8909,7 +8944,7 @@ public class SemanticAnalyzer extends Ba
curr = genWindowingPlan(qb.getWindowingSpec(dest), curr);
}
- curr = genSelectPlan(dest, qb, curr);
+ curr = genSelectPlan(dest, qb, curr, gbySource);
Integer limit = qbp.getDestLimit(dest);
// Expressions are not supported currently without a alias.
@@ -9866,7 +9901,7 @@ public class SemanticAnalyzer extends Ba
// Get the UDTF Path
QB blankQb = new QB(null, null, false);
Operator udtfPath = genSelectPlan((ASTNode) lateralViewTree
- .getChild(0), blankQb, lvForward,
+ .getChild(0), blankQb, lvForward, null,
lateralViewTree.getType() == HiveParser.TOK_LATERAL_VIEW_OUTER);
// add udtf aliases to QB
for (String udtfAlias : blankQb.getAliases()) {
@@ -12415,7 +12450,9 @@ public class SemanticAnalyzer extends Ba
private boolean isAcidTable(Table tab) {
if (tab == null) return false;
if (!SessionState.get().getTxnMgr().supportsAcid()) return false;
- return tab.getProperty(ACID_TABLE_PROPERTY) != null;
+ String tableIsTransactional =
+ tab.getProperty(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+ return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
}
private boolean isAcidOutputFormat(Class<? extends HiveOutputFormat> of) {
@@ -14203,8 +14240,8 @@ public class SemanticAnalyzer extends Ba
if (expr.getType() == HiveParser.TOK_ALLCOLREF) {
pos = genColListRegex(".*",
expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0))
- .toLowerCase(), expr, col_list, excludedColumns, inputRR, pos, out_rwsch,
- tabAliasesForAllProjs, true);
+ .toLowerCase(), expr, col_list, excludedColumns, inputRR, null, pos,
+ out_rwsch, tabAliasesForAllProjs, true);
selectStar = true;
} else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause
&& !inputRR.getIsExprResolver()
@@ -14213,7 +14250,8 @@ public class SemanticAnalyzer extends Ba
// This can only happen without AS clause
// We don't allow this for ExprResolver - the Group By case
pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()), null, expr,
- col_list, excludedColumns, inputRR, pos, out_rwsch, tabAliasesForAllProjs, true);
+ col_list, excludedColumns, inputRR, null, pos, out_rwsch, tabAliasesForAllProjs,
+ true);
} else if (expr.getType() == HiveParser.DOT
&& expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL
&& inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0).getChild(0).getText()
@@ -14224,7 +14262,8 @@ public class SemanticAnalyzer extends Ba
// We don't allow this for ExprResolver - the Group By case
pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()),
unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()), expr,
- col_list, excludedColumns, inputRR, pos, out_rwsch, tabAliasesForAllProjs, true);
+ col_list, excludedColumns, inputRR, null, pos, out_rwsch, tabAliasesForAllProjs,
+ true);
} else if (expr.toStringTree().contains("TOK_FUNCTIONDI") && !(srcRel instanceof HiveAggregateRel)) {
// Likely a malformed query eg, select hash(distinct c1) from t1;
throw new OptiqSemanticException("Distinct without an aggreggation.");
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java Tue Nov 18 00:48:40 2014
@@ -150,6 +150,7 @@ public class TezCompiler extends TaskCom
LOG.info("Found cycle in operator plan...");
cycleFree = false;
removeEventOperator(component);
+ break;
}
}
LOG.info("Cycle free: " + cycleFree);
@@ -227,7 +228,7 @@ public class TezCompiler extends TaskCom
for (Operator<?> child : children) {
if (!indexes.containsKey(child)) {
connect(child, index, nodes, indexes, lowLinks, components);
- lowLinks.put(child, Math.min(lowLinks.get(o), lowLinks.get(child)));
+ lowLinks.put(o, Math.min(lowLinks.get(o), lowLinks.get(child)));
} else if (nodes.contains(child)) {
lowLinks.put(o, Math.min(lowLinks.get(o), indexes.get(child)));
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Tue Nov 18 00:48:40 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.parse;
import java.sql.Date;
+import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -73,6 +74,8 @@ import org.apache.hadoop.hive.serde2.typ
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import com.google.common.collect.Lists;
+
/**
* The Factory for creating typecheck processors. The typecheck processors are
@@ -170,7 +173,8 @@ public class TypeCheckProcFactory {
tf.getStrExprProcessor());
opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|"
+ HiveParser.KW_FALSE + "%"), tf.getBoolExprProcessor());
- opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%"), tf.getDateExprProcessor());
+ opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%|"
+ + HiveParser.TOK_TIMESTAMPLITERAL + "%"), tf.getDateTimeExprProcessor());
opRules.put(new RuleRegExp("R6", HiveParser.TOK_TABLE_OR_COL + "%"),
tf.getColumnExprProcessor());
opRules.put(new RuleRegExp("R7", HiveParser.TOK_SUBQUERY_OP + "%"),
@@ -182,9 +186,8 @@ public class TypeCheckProcFactory {
opRules, tcCtx);
GraphWalker ogw = new DefaultGraphWalker(disp);
- // Create a list of topop nodes
- ArrayList<Node> topNodes = new ArrayList<Node>();
- topNodes.add(expr);
+ // Create a list of top nodes
+ ArrayList<Node> topNodes = Lists.<Node>newArrayList(expr);
HashMap<Node, Object> nodeOutputs = new LinkedHashMap<Node, Object>();
ogw.startWalking(topNodes, nodeOutputs);
@@ -420,7 +423,7 @@ public class TypeCheckProcFactory {
/**
* Processor for date constants.
*/
- public static class DateExprProcessor implements NodeProcessor {
+ public static class DateTimeExprProcessor implements NodeProcessor {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
@@ -437,14 +440,24 @@ public class TypeCheckProcFactory {
}
ASTNode expr = (ASTNode) nd;
+ String timeString = BaseSemanticAnalyzer.stripQuotes(expr.getText());
// Get the string value and convert to a Date value.
try {
- String dateString = BaseSemanticAnalyzer.stripQuotes(expr.getText());
- Date date = Date.valueOf(dateString);
- return new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo, date);
- } catch (IllegalArgumentException err) {
- throw new SemanticException("Unable to convert date literal string to date value.", err);
+ // todo replace below with joda-time, which supports timezone
+ if (expr.getType() == HiveParser.TOK_DATELITERAL) {
+ PrimitiveTypeInfo typeInfo = TypeInfoFactory.dateTypeInfo;
+ return new ExprNodeConstantDesc(typeInfo,
+ Date.valueOf(timeString));
+ }
+ if (expr.getType() == HiveParser.TOK_TIMESTAMPLITERAL) {
+ return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo,
+ Timestamp.valueOf(timeString));
+ }
+ throw new IllegalArgumentException("Invalid time literal type " + expr.getType());
+ } catch (Exception err) {
+ throw new SemanticException(
+ "Unable to convert time literal '" + timeString + "' to time value.", err);
}
}
}
@@ -454,8 +467,8 @@ public class TypeCheckProcFactory {
*
* @return DateExprProcessor.
*/
- public DateExprProcessor getDateExprProcessor() {
- return new DateExprProcessor();
+ public DateTimeExprProcessor getDateTimeExprProcessor() {
+ return new DateTimeExprProcessor();
}
/**
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactory.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactory.java Tue Nov 18 00:48:40 2014
@@ -21,6 +21,8 @@ import java.io.Serializable;
import java.util.HashSet;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
@@ -32,6 +34,8 @@ import org.apache.hadoop.hive.ql.parse.S
* tasks. Every method in this class may return null, indicating no task
* needs to be executed or can throw a SemanticException.
*/
+@LimitedPrivate(value = { "Apache Hive, Apache Sentry (incubating)" })
+@Evolving
public interface HiveAuthorizationTaskFactory {
public Task<? extends Serializable> createCreateRoleTask(ASTNode node, HashSet<ReadEntity> inputs,
HashSet<WriteEntity> outputs) throws SemanticException;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java Tue Nov 18 00:48:40 2014
@@ -238,7 +238,7 @@ public class HiveAuthorizationTaskFactor
return subject;
}
- private PrivilegeObjectDesc parsePrivObject(ASTNode ast) throws SemanticException {
+ protected PrivilegeObjectDesc parsePrivObject(ASTNode ast) throws SemanticException {
PrivilegeObjectDesc subject = new PrivilegeObjectDesc();
ASTNode child = (ASTNode) ast.getChild(0);
ASTNode gchild = (ASTNode)child.getChild(0);
@@ -246,6 +246,8 @@ public class HiveAuthorizationTaskFactor
subject.setTable(true);
String[] qualified = BaseSemanticAnalyzer.getQualifiedTableName(gchild);
subject.setObject(BaseSemanticAnalyzer.getDotName(qualified));
+ } else if (child.getType() == HiveParser.TOK_URI_TYPE || child.getType() == HiveParser.TOK_SERVER_TYPE) {
+ throw new SemanticException("Hive authorization does not support the URI or SERVER objects");
} else {
subject.setTable(false);
subject.setObject(BaseSemanticAnalyzer.unescapeIdentifier(gchild.getText()));
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java Tue Nov 18 00:48:40 2014
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.plan;
import java.util.ArrayList;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java Tue Nov 18 00:48:40 2014
@@ -134,6 +134,7 @@ public class JDBCStatsAggregator impleme
}
};
+ fileID = JDBCStatsUtils.truncateRowId(fileID);
String keyPrefix = Utilities.escapeSqlLike(fileID) + "%";
for (int failures = 0;; failures++) {
try {
@@ -147,7 +148,7 @@ public class JDBCStatsAggregator impleme
if (result.next()) {
retval = result.getLong(1);
} else {
- LOG.warn("Warning. Nothing published. Nothing aggregated.");
+ LOG.warn("Nothing published. Nothing aggregated.");
return null;
}
return Long.toString(retval);
@@ -217,6 +218,7 @@ public class JDBCStatsAggregator impleme
};
try {
+ rowID = JDBCStatsUtils.truncateRowId(rowID);
String keyPrefix = Utilities.escapeSqlLike(rowID) + "%";
PreparedStatement delStmt = Utilities.prepareWithRetry(conn,
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java Tue Nov 18 00:48:40 2014
@@ -139,7 +139,11 @@ public class JDBCStatsPublisher implemen
+ " stats: " + JDBCStatsUtils.getSupportedStatistics());
return false;
}
- LOG.info("Stats publishing for key " + fileID);
+ String rowId = JDBCStatsUtils.truncateRowId(fileID);
+ if (LOG.isInfoEnabled()) {
+ String truncateSuffix = (rowId != fileID) ? " (from " + fileID + ")" : ""; // object equality
+ LOG.info("Stats publishing for key " + rowId + truncateSuffix);
+ }
Utilities.SQLCommand<Void> execUpdate = new Utilities.SQLCommand<Void>() {
@Override
@@ -153,7 +157,7 @@ public class JDBCStatsPublisher implemen
for (int failures = 0;; failures++) {
try {
- insStmt.setString(1, fileID);
+ insStmt.setString(1, rowId);
for (int i = 0; i < JDBCStatsUtils.getSupportedStatistics().size(); i++) {
insStmt.setString(i + 2, stats.get(supportedStatistics.get(i)));
}
@@ -172,10 +176,10 @@ public class JDBCStatsPublisher implemen
for (i = 0; i < JDBCStatsUtils.getSupportedStatistics().size(); i++) {
updStmt.setString(i + 1, stats.get(supportedStatistics.get(i)));
}
- updStmt.setString(supportedStatistics.size() + 1, fileID);
+ updStmt.setString(supportedStatistics.size() + 1, rowId);
updStmt.setString(supportedStatistics.size() + 2,
stats.get(JDBCStatsUtils.getBasicStat()));
- updStmt.setString(supportedStatistics.size() + 3, fileID);
+ updStmt.setString(supportedStatistics.size() + 3, rowId);
Utilities.executeWithRetry(execUpdate, updStmt, waitWindow, maxRetries);
return true;
} catch (SQLRecoverableException ue) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java Tue Nov 18 00:48:40 2014
@@ -34,4 +34,7 @@ public final class JDBCStatsSetupConstan
public static final String PART_STAT_RAW_DATA_SIZE_COLUMN_NAME = "RAW_DATA_SIZE";
+ // 255 is an old value that we will keep for now; it can be increased to 4000; limits are
+ // MySQL - 65535, SQL Server - 8000, Oracle - 4000, Derby - 32762, Postgres - large.
+ public static final int ID_COLUMN_VARCHAR_SIZE = 255;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java Tue Nov 18 00:48:40 2014
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.util.hash.MurmurHash;
public class JDBCStatsUtils {
@@ -124,9 +125,10 @@ public class JDBCStatsUtils {
* Prepares CREATE TABLE query
*/
public static String getCreate(String comment) {
- String create = "CREATE TABLE /* " + comment + " */ " + JDBCStatsUtils.getStatTableName() +
- " (" + getTimestampColumnName() + " TIMESTAMP DEFAULT CURRENT_TIMESTAMP, " +
- JDBCStatsUtils.getIdColumnName() + " VARCHAR(255) PRIMARY KEY ";
+ String create = "CREATE TABLE /* " + comment + " */ " + JDBCStatsUtils.getStatTableName()
+ + " (" + getTimestampColumnName() + " TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "
+ + JDBCStatsUtils.getIdColumnName() + " VARCHAR("
+ + JDBCStatsSetupConstants.ID_COLUMN_VARCHAR_SIZE + ") PRIMARY KEY ";
for (int i = 0; i < supportedStats.size(); i++) {
create += ", " + getStatColumnName(supportedStats.get(i)) + " BIGINT ";
}
@@ -191,4 +193,13 @@ public class JDBCStatsUtils {
return delete;
}
+ /**
+ * Make sure the row ID fits into the row ID column in the table.
+ * @param rowId Row ID.
+ * @return Resulting row ID truncated to correct length, if necessary.
+ */
+ public static String truncateRowId(String rowId) {
+ return (rowId.length() <= JDBCStatsSetupConstants.ID_COLUMN_VARCHAR_SIZE)
+ ? rowId : Integer.toHexString(MurmurHash.getInstance().hash(rowId.getBytes()));
+ }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java Tue Nov 18 00:48:40 2014
@@ -69,7 +69,7 @@ public class Cleaner extends CompactorTh
// This is solely for testing. It checks if the test has set the looped value to false,
// and if so remembers that and then sets it to true at the end. We have to check here
// first to make sure we go through a complete iteration of the loop before resetting it.
- boolean setLooped = !looped.boolVal;
+ boolean setLooped = !looped.get();
// Make sure nothing escapes this run method and kills the metastore at large,
// so wrap it in a big catch Throwable statement.
try {
@@ -137,16 +137,16 @@ public class Cleaner extends CompactorTh
// Now, go back to bed until it's time to do this again
long elapsedTime = System.currentTimeMillis() - startedAt;
- if (elapsedTime >= cleanerCheckInterval || stop.boolVal) continue;
+ if (elapsedTime >= cleanerCheckInterval || stop.get()) continue;
else Thread.sleep(cleanerCheckInterval - elapsedTime);
} catch (Throwable t) {
LOG.error("Caught an exception in the main loop of compactor cleaner, " +
StringUtils.stringifyException(t));
}
if (setLooped) {
- looped.boolVal = true;
+ looped.set(true);
}
- } while (!stop.boolVal);
+ } while (!stop.get());
}
private Set<Long> findRelatedLocks(CompactionInfo ci, ShowLocksResponse locksResponse) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java Tue Nov 18 00:48:40 2014
@@ -40,6 +40,7 @@ import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* Superclass for all threads in the compactor.
@@ -52,8 +53,8 @@ abstract class CompactorThread extends T
protected CompactionTxnHandler txnHandler;
protected RawStore rs;
protected int threadId;
- protected BooleanPointer stop;
- protected BooleanPointer looped;
+ protected AtomicBoolean stop;
+ protected AtomicBoolean looped;
@Override
public void setHiveConf(HiveConf conf) {
@@ -67,7 +68,7 @@ abstract class CompactorThread extends T
}
@Override
- public void init(BooleanPointer stop, BooleanPointer looped) throws MetaException {
+ public void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException {
this.stop = stop;
this.looped = looped;
setPriority(MIN_PRIORITY);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java Tue Nov 18 00:48:40 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
import org.apache.hadoop.hive.metastore.txn.TxnHandler;
import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -44,6 +45,7 @@ import java.security.PrivilegedException
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* A class to initiate compactions. This will run in a separate thread.
@@ -52,8 +54,6 @@ public class Initiator extends Compactor
static final private String CLASS_NAME = Initiator.class.getName();
static final private Log LOG = LogFactory.getLog(CLASS_NAME);
- static final private String NO_COMPACTION = "NO_AUTO_COMPACTION";
-
private long checkInterval;
@Override
@@ -85,9 +85,8 @@ public class Initiator extends Compactor
try {
Table t = resolveTable(ci);
// check if no compaction set for this table
- if (t.getParameters().get(NO_COMPACTION) != null) {
- LOG.info("Table " + tableName(t) + " marked " + NO_COMPACTION +
- " so we will not compact it.");
+ if (noAutoCompactSet(t)) {
+ LOG.info("Table " + tableName(t) + " marked true so we will not compact it.");
continue;
}
@@ -126,10 +125,10 @@ public class Initiator extends Compactor
}
long elapsedTime = System.currentTimeMillis() - startedAt;
- if (elapsedTime >= checkInterval || stop.boolVal) continue;
+ if (elapsedTime >= checkInterval || stop.get()) continue;
else Thread.sleep(checkInterval - elapsedTime);
- } while (!stop.boolVal);
+ } while (!stop.get());
} catch (Throwable t) {
LOG.error("Caught an exception in the main loop of compactor initiator, exiting " +
StringUtils.stringifyException(t));
@@ -137,7 +136,7 @@ public class Initiator extends Compactor
}
@Override
- public void init(BooleanPointer stop, BooleanPointer looped) throws MetaException {
+ public void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException {
super.init(stop, looped);
checkInterval =
conf.getTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL, TimeUnit.MILLISECONDS) ;
@@ -278,4 +277,16 @@ public class Initiator extends Compactor
rqst.setRunas(runAs);
txnHandler.compact(rqst);
}
+
+ // Because TABLE_NO_AUTO_COMPACT was originally assumed to be NO_AUTO_COMPACT and then was moved
+ // to no_auto_compact, we need to check it in both cases.
+ private boolean noAutoCompactSet(Table t) {
+ String noAutoCompact =
+ t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT);
+ if (noAutoCompact == null) {
+ noAutoCompact =
+ t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT.toUpperCase());
+ }
+ return noAutoCompact != null && noAutoCompact.equalsIgnoreCase("true");
+ }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java Tue Nov 18 00:48:40 2014
@@ -42,6 +42,7 @@ import java.security.PrivilegedException
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* A class to do compactions. This will run in a separate thread. It will spin on the
@@ -77,7 +78,7 @@ public class Worker extends CompactorThr
do {
CompactionInfo ci = txnHandler.findNextToCompact(name);
- if (ci == null && !stop.boolVal) {
+ if (ci == null && !stop.get()) {
try {
Thread.sleep(SLEEP_TIME);
continue;
@@ -160,7 +161,7 @@ public class Worker extends CompactorThr
". Marking clean to avoid repeated failures, " + StringUtils.stringifyException(e));
txnHandler.markCleaned(ci);
}
- } while (!stop.boolVal);
+ } while (!stop.get());
} catch (Throwable t) {
LOG.error("Caught an exception in the main loop of compactor worker " + name +
", exiting " + StringUtils.stringifyException(t));
@@ -168,7 +169,7 @@ public class Worker extends CompactorThr
}
@Override
- public void init(BooleanPointer stop, BooleanPointer looped) throws MetaException {
+ public void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException {
super.init(stop, looped);
StringBuilder name = new StringBuilder(hostname());
Modified: hive/branches/spark/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto (original)
+++ hive/branches/spark/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto Tue Nov 18 00:48:40 2014
@@ -191,8 +191,15 @@ message PostScript {
optional uint64 footerLength = 1;
optional CompressionKind compression = 2;
optional uint64 compressionBlockSize = 3;
+ // the version of the file format
+ // [0, 11] = Hive 0.11
+ // [0, 12] = Hive 0.12
repeated uint32 version = 4 [packed = true];
optional uint64 metadataLength = 5;
+ // Version of the writer:
+ // 0 (or missing) = original
+ // 1 = HIVE-8732 fixed
+ optional uint32 writerVersion = 6;
// Leave this last in the record
optional string magic = 8000;
}
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java Tue Nov 18 00:48:40 2014
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.exec;
import org.apache.commons.logging.Log;
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java Tue Nov 18 00:48:40 2014
@@ -38,8 +38,9 @@ public class TestDecimalUtil {
DecimalUtil.floor(0, d1, dcv);
Assert.assertEquals(0, expected1.compareTo(dcv.vector[0].getHiveDecimal()));
+ // As of HIVE-8745, these decimal values should be trimmed of trailing zeros.
HiveDecimal d2 = HiveDecimal.create("23.00000");
- Assert.assertEquals(5, d2.scale());
+ Assert.assertEquals(0, d2.scale());
HiveDecimal expected2 = HiveDecimal.create("23");
DecimalUtil.floor(0, d2, dcv);
Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -50,19 +51,19 @@ public class TestDecimalUtil {
Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal()));
HiveDecimal d4 = HiveDecimal.create("-17.00000");
- Assert.assertEquals(5, d4.scale());
+ Assert.assertEquals(0, d4.scale());
HiveDecimal expected4 = HiveDecimal.create("-17");
DecimalUtil.floor(0, d4, dcv);
Assert.assertEquals(0, expected4.compareTo(dcv.vector[0].getHiveDecimal()));
HiveDecimal d5 = HiveDecimal.create("-0.30000");
- Assert.assertEquals(5, d5.scale());
+ Assert.assertEquals(1, d5.scale());
HiveDecimal expected5 = HiveDecimal.create("-1");
DecimalUtil.floor(0, d5, dcv);
Assert.assertEquals(0, expected5.compareTo(dcv.vector[0].getHiveDecimal()));
HiveDecimal d6 = HiveDecimal.create("0.30000");
- Assert.assertEquals(5, d6.scale());
+ Assert.assertEquals(1, d6.scale());
HiveDecimal expected6 = HiveDecimal.create("0");
DecimalUtil.floor(0, d6, dcv);
Assert.assertEquals(0, expected6.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -76,8 +77,9 @@ public class TestDecimalUtil {
DecimalUtil.ceiling(0, d1, dcv);
Assert.assertEquals(0, expected1.compareTo(dcv.vector[0].getHiveDecimal()));
+ // As of HIVE-8745, these decimal values should be trimmed of trailing zeros.
HiveDecimal d2 = HiveDecimal.create("23.00000");
- Assert.assertEquals(5, d2.scale());
+ Assert.assertEquals(0, d2.scale());
HiveDecimal expected2 = HiveDecimal.create("23");
DecimalUtil.ceiling(0, d2, dcv);
Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -88,19 +90,19 @@ public class TestDecimalUtil {
Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal()));
HiveDecimal d4 = HiveDecimal.create("-17.00000");
- Assert.assertEquals(5, d4.scale());
+ Assert.assertEquals(0, d4.scale());
HiveDecimal expected4 = HiveDecimal.create("-17");
DecimalUtil.ceiling(0, d4, dcv);
Assert.assertEquals(0, expected4.compareTo(dcv.vector[0].getHiveDecimal()));
HiveDecimal d5 = HiveDecimal.create("-0.30000");
- Assert.assertEquals(5, d5.scale());
+ Assert.assertEquals(1, d5.scale());
HiveDecimal expected5 = HiveDecimal.create("0");
DecimalUtil.ceiling(0, d5, dcv);
Assert.assertEquals(0, expected5.compareTo(dcv.vector[0].getHiveDecimal()));
HiveDecimal d6 = HiveDecimal.create("0.30000");
- Assert.assertEquals(5, d6.scale());
+ Assert.assertEquals(1, d6.scale());
HiveDecimal expected6 = HiveDecimal.create("1");
DecimalUtil.ceiling(0, d6, dcv);
Assert.assertEquals(0, expected6.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -127,8 +129,9 @@ public class TestDecimalUtil {
DecimalUtil.round(0, d1, dcv);
Assert.assertEquals(0, expected1.compareTo(dcv.vector[0].getHiveDecimal()));
+ // As of HIVE-8745, these decimal values should be trimmed of trailing zeros.
HiveDecimal d2 = HiveDecimal.create("23.00000");
- Assert.assertEquals(5, d2.scale());
+ Assert.assertEquals(0, d2.scale());
HiveDecimal expected2 = HiveDecimal.create("23");
DecimalUtil.round(0, d2, dcv);
Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -139,7 +142,7 @@ public class TestDecimalUtil {
Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal()));
HiveDecimal d4 = HiveDecimal.create("-17.00000");
- Assert.assertEquals(5, d4.scale());
+ Assert.assertEquals(0, d4.scale());
HiveDecimal expected4 = HiveDecimal.create("-17");
DecimalUtil.round(0, d4, dcv);
Assert.assertEquals(0, expected4.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -163,8 +166,9 @@ public class TestDecimalUtil {
DecimalUtil.round(0, d1, dcv);
Assert.assertEquals(0, expected1.compareTo(dcv.vector[0].getHiveDecimal()));
+ // As of HIVE-8745, these decimal values should be trimmed of trailing zeros.
HiveDecimal d2 = HiveDecimal.create("23.56700");
- Assert.assertEquals(5, d2.scale());
+ Assert.assertEquals(3, d2.scale());
HiveDecimal expected2 = HiveDecimal.create("23.567");
DecimalUtil.round(0, d2, dcv);
Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -175,7 +179,7 @@ public class TestDecimalUtil {
Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal()));
HiveDecimal d4 = HiveDecimal.create("-17.23400");
- Assert.assertEquals(5, d4.scale());
+ Assert.assertEquals(3, d4.scale());
HiveDecimal expected4 = HiveDecimal.create("-17.234");
DecimalUtil.round(0, d4, dcv);
Assert.assertEquals(0, expected4.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -204,8 +208,9 @@ public class TestDecimalUtil {
DecimalUtil.negate(0, d2, dcv);
Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal()));
+ // As of HIVE-8745, these decimal values should be trimmed of trailing zeros.
HiveDecimal d3 = HiveDecimal.create("0.00000");
- Assert.assertEquals(5, d3.scale());
+ Assert.assertEquals(0, d3.scale());
HiveDecimal expected3 = HiveDecimal.create("0");
DecimalUtil.negate(0, d3, dcv);
Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -223,7 +228,7 @@ public class TestDecimalUtil {
Assert.assertEquals(-1, lcv.vector[0]);
HiveDecimal d3 = HiveDecimal.create("0.00000");
- Assert.assertEquals(5, d3.scale());
+ Assert.assertEquals(0, d3.scale());
d3.setScale(5);
DecimalUtil.sign(0, d3, lcv);
Assert.assertEquals(0, lcv.vector[0]);
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java Tue Nov 18 00:48:40 2014
@@ -323,18 +323,19 @@ public class TestVectorTypeCasts {
expr.evaluate(b);
BytesColumnVector r = (BytesColumnVector) b.cols[1];
- byte[] v = toBytes("1.10");
+ // As of HIVE-8745, these decimal values should be trimmed of trailing zeros.
+ byte[] v = toBytes("1.1");
assertTrue(((Integer) v.length).toString() + " " + r.length[0], v.length == r.length[0]);
Assert.assertEquals(0,
StringExpr.compare(v, 0, v.length,
r.vector[0], r.start[0], r.length[0]));
- v = toBytes("-2.20");
+ v = toBytes("-2.2");
Assert.assertEquals(0,
StringExpr.compare(v, 0, v.length,
r.vector[1], r.start[1], r.length[1]));
- v = toBytes("9999999999999999.00");
+ v = toBytes("9999999999999999");
Assert.assertEquals(0,
StringExpr.compare(v, 0, v.length,
r.vector[2], r.start[2], r.length[2]));
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java Tue Nov 18 00:48:40 2014
@@ -21,27 +21,6 @@ import static org.junit.Assert.assertArr
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.sql.Date;
-import java.sql.Timestamp;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.TreeSet;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -66,9 +45,9 @@ import org.apache.hadoop.hive.ql.io.Comb
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.io.InputFormatChecker;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
@@ -104,6 +83,27 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.TreeSet;
+
public class TestInputOutputFormat {
Path workDir = new Path(System.getProperty("test.tmp.dir","target/tmp"));
@@ -1032,6 +1032,24 @@ public class TestInputOutputFormat {
reader.close();
}
+ static class SimpleRow implements Writable {
+ Text z;
+
+ public SimpleRow(Text t) {
+ this.z = t;
+ }
+
+ @Override
+ public void write(DataOutput dataOutput) throws IOException {
+ throw new UnsupportedOperationException("unsupported");
+ }
+
+ @Override
+ public void readFields(DataInput dataInput) throws IOException {
+ throw new UnsupportedOperationException("unsupported");
+ }
+ }
+
static class NestedRow implements Writable {
int z;
MyRow r;
@@ -1620,14 +1638,14 @@ public class TestInputOutputFormat {
assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00000",
split.getPath().toString());
assertEquals(0, split.getStart());
- assertEquals(580, split.getLength());
+ assertEquals(582, split.getLength());
split = (HiveInputFormat.HiveInputSplit) splits[1];
assertEquals("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
split.inputFormatClassName());
assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00001",
split.getPath().toString());
assertEquals(0, split.getStart());
- assertEquals(601, split.getLength());
+ assertEquals(603, split.getLength());
CombineHiveInputFormat.CombineHiveInputSplit combineSplit =
(CombineHiveInputFormat.CombineHiveInputSplit) splits[2];
assertEquals(BUCKETS, combineSplit.getNumPaths());
@@ -1635,7 +1653,7 @@ public class TestInputOutputFormat {
assertEquals("mock:/combinationAcid/p=1/00000" + bucket + "_0",
combineSplit.getPath(bucket).toString());
assertEquals(0, combineSplit.getOffset(bucket));
- assertEquals(225, combineSplit.getLength(bucket));
+ assertEquals(227, combineSplit.getLength(bucket));
}
String[] hosts = combineSplit.getLocations();
assertEquals(2, hosts.length);
@@ -1685,4 +1703,89 @@ public class TestInputOutputFormat {
assertEquals("cost", leaves.get(0).getColumnName());
assertEquals(PredicateLeaf.Operator.IS_NULL, leaves.get(0).getOperator());
}
+
+ @Test
+ @SuppressWarnings("unchecked,deprecation")
+ public void testSplitElimination() throws Exception {
+ Properties properties = new Properties();
+ StructObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = (StructObjectInspector)
+ ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ SerDe serde = new OrcSerde();
+ OutputFormat<?, ?> outFormat = new OrcOutputFormat();
+ conf.setInt("mapred.max.split.size", 50);
+ RecordWriter writer =
+ outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
+ Reporter.NULL);
+ writer.write(NullWritable.get(),
+ serde.serialize(new NestedRow(1,2,3), inspector));
+ writer.write(NullWritable.get(),
+ serde.serialize(new NestedRow(4,5,6), inspector));
+ writer.write(NullWritable.get(),
+ serde.serialize(new NestedRow(7,8,9), inspector));
+ writer.close(Reporter.NULL);
+ serde = new OrcSerde();
+ SearchArgument sarg =
+ SearchArgumentFactory.newBuilder()
+ .startAnd()
+ .lessThan("z", new Integer(0))
+ .end()
+ .build();
+ conf.set("sarg.pushdown", sarg.toKryo());
+ conf.set("hive.io.file.readcolumn.names", "z,r");
+ properties.setProperty("columns", "z,r");
+ properties.setProperty("columns.types", "int:struct<x:int,y:int>");
+ SerDeUtils.initializeSerDe(serde, conf, properties, null);
+ inspector = (StructObjectInspector) serde.getObjectInspector();
+ InputFormat<?,?> in = new OrcInputFormat();
+ FileInputFormat.setInputPaths(conf, testFilePath.toString());
+ InputSplit[] splits = in.getSplits(conf, 1);
+ assertEquals(0, splits.length);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked,deprecation")
+ public void testSplitEliminationNullStats() throws Exception {
+ Properties properties = new Properties();
+ StructObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = (StructObjectInspector)
+ ObjectInspectorFactory.getReflectionObjectInspector(SimpleRow.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ SerDe serde = new OrcSerde();
+ OutputFormat<?, ?> outFormat = new OrcOutputFormat();
+ conf.setInt("mapred.max.split.size", 50);
+ RecordWriter writer =
+ outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
+ Reporter.NULL);
+ writer.write(NullWritable.get(),
+ serde.serialize(new SimpleRow(null), inspector));
+ writer.write(NullWritable.get(),
+ serde.serialize(new SimpleRow(null), inspector));
+ writer.write(NullWritable.get(),
+ serde.serialize(new SimpleRow(null), inspector));
+ writer.close(Reporter.NULL);
+ serde = new OrcSerde();
+ SearchArgument sarg =
+ SearchArgumentFactory.newBuilder()
+ .startAnd()
+ .lessThan("z", new String("foo"))
+ .end()
+ .build();
+ conf.set("sarg.pushdown", sarg.toKryo());
+ conf.set("hive.io.file.readcolumn.names", "z");
+ properties.setProperty("columns", "z");
+ properties.setProperty("columns.types", "string");
+ SerDeUtils.initializeSerDe(serde, conf, properties, null);
+ inspector = (StructObjectInspector) serde.getObjectInspector();
+ InputFormat<?,?> in = new OrcInputFormat();
+ FileInputFormat.setInputPaths(conf, testFilePath.toString());
+ InputSplit[] splits = in.getSplits(conf, 1);
+ assertEquals(0, splits.length);
+ }
+
}
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetByteInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetByteInspector.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetByteInspector.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetByteInspector.java Tue Nov 18 00:48:40 2014
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.io.parquet.serde.primitive;
import static org.junit.Assert.assertEquals;
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetShortInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetShortInspector.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetShortInspector.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetShortInspector.java Tue Nov 18 00:48:40 2014
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.io.parquet.serde.primitive;
import static org.junit.Assert.assertEquals;
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java Tue Nov 18 00:48:40 2014
@@ -2830,7 +2830,7 @@ public class TestSearchArgumentImpl {
.build();
assertEquals("leaf-0 = (LESS_THAN x 1970-01-11)\n" +
"leaf-1 = (LESS_THAN_EQUALS y hi)\n" +
- "leaf-2 = (EQUALS z 1.0)\n" +
+ "leaf-2 = (EQUALS z 1)\n" +
"expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString());
sarg = SearchArgumentFactory.newBuilder()
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java Tue Nov 18 00:48:40 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FSDataInputS
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.exec.ExplainTask;
@@ -267,7 +268,7 @@ public class TestUpdateDeleteSemanticAna
// I have to create the tables here (rather than in setup()) because I need the Hive
// connection, which is conviently created by the semantic analyzer.
Map<String, String> params = new HashMap<String, String>(1);
- params.put(SemanticAnalyzer.ACID_TABLE_PROPERTY, "true");
+ params.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
db.createTable("T", Arrays.asList("a", "b"), null, OrcInputFormat.class,
OrcOutputFormat.class, 2, Arrays.asList("a"), params);
db.createTable("U", Arrays.asList("a", "b"), Arrays.asList("ds"), OrcInputFormat.class,
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java Tue Nov 18 00:48:40 2014
@@ -22,18 +22,23 @@ import java.util.HashMap;
import junit.framework.Assert;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.ParseDriver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.plan.GrantDesc;
import org.apache.hadoop.hive.ql.plan.GrantRevokeRoleDDL;
import org.apache.hadoop.hive.ql.plan.PrincipalDesc;
import org.apache.hadoop.hive.ql.plan.PrivilegeDesc;
+import org.apache.hadoop.hive.ql.plan.PrivilegeObjectDesc;
import org.apache.hadoop.hive.ql.plan.RevokeDesc;
import org.apache.hadoop.hive.ql.plan.RoleDDLDesc;
import org.apache.hadoop.hive.ql.plan.RoleDDLDesc.RoleOperation;
@@ -47,6 +52,33 @@ import org.mockito.Mockito;
public class TestHiveAuthorizationTaskFactory {
+ public static class DummyHiveAuthorizationTaskFactoryImpl extends HiveAuthorizationTaskFactoryImpl {
+
+ static String uriPath = "";
+ static String serverName = "";
+
+ public DummyHiveAuthorizationTaskFactoryImpl(HiveConf conf, Hive db) {
+ super(conf, db);
+ }
+
+ @Override
+ protected PrivilegeObjectDesc parsePrivObject(ASTNode ast) throws SemanticException {
+ ASTNode child = (ASTNode) ast.getChild(0);
+ ASTNode gchild = (ASTNode)child.getChild(0);
+ if (child.getType() == HiveParser.TOK_URI_TYPE) {
+ uriPath = gchild.getText().replaceAll("'", "").replaceAll("\"", "");
+ } else if (child.getType() == HiveParser.TOK_SERVER_TYPE) {
+ serverName = gchild.getText();
+ }
+ return super.parsePrivObject(ast);
+ }
+
+ public static void reset() {
+ uriPath = "";
+ serverName = "";
+ }
+ }
+
private static final String SELECT = "SELECT";
private static final String DB = "default";
private static final String TABLE = "table1";
@@ -67,6 +99,8 @@ public class TestHiveAuthorizationTaskFa
@Before
public void setup() throws Exception {
conf = new HiveConf();
+ conf.setVar(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY,
+ TestHiveAuthorizationTaskFactory.DummyHiveAuthorizationTaskFactoryImpl.class.getName());
db = Mockito.mock(Hive.class);
table = new Table(DB, TABLE);
partition = new Partition(table);
@@ -81,6 +115,7 @@ public class TestHiveAuthorizationTaskFa
HadoopDefaultAuthenticator auth = new HadoopDefaultAuthenticator();
auth.setConf(conf);
currentUser = auth.getUserName();
+ DummyHiveAuthorizationTaskFactoryImpl.reset();
}
/**
@@ -414,6 +449,34 @@ public class TestHiveAuthorizationTaskFa
Assert.assertTrue("Expected table", grantDesc.getHiveObj().getTable());
}
+ /**
+ * GRANT ALL ON URI
+ */
+ @Test
+ public void testGrantUri() throws Exception {
+ String uriPath = "/tmp";
+ try {
+ analyze("GRANT ALL ON URI '" + uriPath + "' TO USER user2");
+ Assert.fail("Grant on URI should fail");
+ } catch (SemanticException e) {
+ Assert.assertEquals(uriPath, DummyHiveAuthorizationTaskFactoryImpl.uriPath);
+ }
+ }
+
+ /**
+ * GRANT ALL ON SERVER
+ */
+ @Test
+ public void testGrantServer() throws Exception {
+ String serverName = "foo";
+ try {
+ analyze("GRANT ALL ON SERVER " + serverName + " TO USER user2");
+ Assert.fail("Grant on Server should fail");
+ } catch (SemanticException e) {
+ Assert.assertEquals(serverName, DummyHiveAuthorizationTaskFactoryImpl.serverName);
+ }
+ }
+
private DDLWork analyze(String command) throws Exception {
return AuthorizationTestUtil.analyze(command, conf, db);
}