You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/07/09 05:06:26 UTC
svn commit: r1608997 [3/5] - in /hive/branches/cbo: ./
beeline/src/java/org/apache/hive/beeline/ beeline/src/main/resources/
bin/ext/ cli/src/java/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/had...
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java Wed Jul 9 03:06:21 2014
@@ -44,7 +44,6 @@ import org.apache.hadoop.hive.ql.hooks.R
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils;
import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc;
import org.apache.hadoop.hive.ql.plan.ColumnStatsWork;
@@ -170,17 +169,15 @@ public abstract class TaskCompiler {
String loc = qb.getTableDesc().getLocation();
if (loc == null) {
// get the table's default location
- Table dumpTable;
Path targetPath;
try {
- dumpTable = db.newTable(qb.getTableDesc().getTableName());
- if (!db.databaseExists(dumpTable.getDbName())) {
- throw new SemanticException("ERROR: The database " + dumpTable.getDbName()
+ String[] names = Utilities.getDbTableName(qb.getTableDesc().getTableName());
+ if (!db.databaseExists(names[0])) {
+ throw new SemanticException("ERROR: The database " + names[0]
+ " does not exist.");
}
Warehouse wh = new Warehouse(conf);
- targetPath = wh.getTablePath(db.getDatabase(dumpTable.getDbName()), dumpTable
- .getTableName());
+ targetPath = wh.getTablePath(db.getDatabase(names[0]), names[1]);
} catch (HiveException e) {
throw new SemanticException(e);
} catch (MetaException e) {
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java Wed Jul 9 03:06:21 2014
@@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.optimiz
import org.apache.hadoop.hive.ql.optimizer.ReduceSinkMapJoinProc;
import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism;
import org.apache.hadoop.hive.ql.optimizer.physical.CrossProductCheck;
+import org.apache.hadoop.hive.ql.optimizer.physical.MetadataOnlyOptimizer;
import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext;
import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer;
import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
@@ -247,15 +248,28 @@ public class TezCompiler extends TaskCom
PhysicalContext physicalCtx = new PhysicalContext(conf, pCtx, pCtx.getContext(), rootTasks,
pCtx.getFetchTask());
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEMETADATAONLYQUERIES)) {
+ physicalCtx = new MetadataOnlyOptimizer().resolve(physicalCtx);
+ } else {
+ LOG.debug("Skipping metadata only query optimization");
+ }
+
if (conf.getBoolVar(HiveConf.ConfVars.HIVE_CHECK_CROSS_PRODUCT)) {
physicalCtx = new CrossProductCheck().resolve(physicalCtx);
+ } else {
+ LOG.debug("Skipping cross product analysis");
}
if (conf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)) {
- (new Vectorizer()).resolve(physicalCtx);
+ physicalCtx = new Vectorizer().resolve(physicalCtx);
+ } else {
+ LOG.debug("Skipping vectorization");
}
+
if (!"none".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVESTAGEIDREARRANGE))) {
- (new StageIDsRearranger()).resolve(physicalCtx);
+ physicalCtx = new StageIDsRearranger().resolve(physicalCtx);
+ } else {
+ LOG.debug("Skipping stage id rearranger");
}
return;
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Wed Jul 9 03:06:21 2014
@@ -987,7 +987,7 @@ public final class TypeCheckProcFactory
// descendant nodes, DFS traversal ensures that the error only needs to
// be cleared once. Also, for a case like
// SELECT concat(value, concat(value))... the logic still works as the
- // error is only set with the first 'value'; all node pocessors quit
+ // error is only set with the first 'value'; all node processors quit
// early if the global error is set.
if (isDescendant(nd, ctx.getErrorSrcNode())) {
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeNullDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeNullDesc.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeNullDesc.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeNullDesc.java Wed Jul 9 03:06:21 2014
@@ -41,6 +41,11 @@ public class ExprNodeNullDesc extends Ex
}
@Override
+ public String toString() {
+ return "null";
+ }
+
+ @Override
public String getExprString() {
return "null";
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java Wed Jul 9 03:06:21 2014
@@ -114,6 +114,8 @@ public class MapWork extends BaseWork {
private boolean useBucketizedHiveInputFormat;
+ private boolean useOneNullRowInputFormat;
+
private Map<String, Map<Integer, String>> scratchColumnVectorTypes = null;
private Map<String, Map<String, Integer>> scratchColumnMap = null;
private boolean vectorMode = false;
@@ -390,6 +392,7 @@ public class MapWork extends BaseWork {
public void setInputformat(String inputformat) {
this.inputformat = inputformat;
}
+
public boolean isUseBucketizedHiveInputFormat() {
return useBucketizedHiveInputFormat;
}
@@ -398,6 +401,14 @@ public class MapWork extends BaseWork {
this.useBucketizedHiveInputFormat = useBucketizedHiveInputFormat;
}
+ public void setUseOneNullRowInputFormat(boolean useOneNullRowInputFormat) {
+ this.useOneNullRowInputFormat = useOneNullRowInputFormat;
+ }
+
+ public boolean isUseOneNullRowInputFormat() {
+ return useOneNullRowInputFormat;
+ }
+
public QBJoinTree getJoinTree() {
return joinTree;
}
@@ -534,4 +545,14 @@ public class MapWork extends BaseWork {
this.vectorMode = vectorMode;
}
+ public void logPathToAliases() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("LOGGING PATH TO ALIASES");
+ for (Map.Entry<String, ArrayList<String>> entry: pathToAliases.entrySet()) {
+ for (String a: entry.getValue()) {
+ LOG.debug("Path: " + entry.getKey() + ", Alias: " + a);
+ }
+ }
+ }
+ }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java Wed Jul 9 03:06:21 2014
@@ -353,6 +353,7 @@ public class ReduceSinkDesc extends Abst
return skipTag;
}
+ @Explain(displayName = "auto parallelism", normalExplain = false)
public final boolean isAutoParallel() {
return (autoParallel != null) && autoParallel;
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java Wed Jul 9 03:06:21 2014
@@ -77,9 +77,7 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFRank.GenericUDAFRankEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan;
-import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction;
import org.apache.hadoop.hive.serde2.Deserializer;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.mapred.JobConf;
@@ -196,7 +194,7 @@ public final class OpProcFactory {
return;
}
- ExprWalkerInfo childInfo = getChildWalkerInfo((Operator<?>) ptfOp, owi);
+ ExprWalkerInfo childInfo = getChildWalkerInfo(ptfOp, owi);
if (childInfo == null) {
return;
@@ -411,16 +409,18 @@ public final class OpProcFactory {
Object... nodeOutputs) throws SemanticException {
LOG.info("Processing for " + nd.getName() + "("
+ ((Operator) nd).getIdentifier() + ")");
+
OpWalkerInfo owi = (OpWalkerInfo) procCtx;
- Operator<? extends OperatorDesc> op =
- (Operator<? extends OperatorDesc>) nd;
- ExprNodeDesc predicate = (((FilterOperator) nd).getConf()).getPredicate();
- ExprWalkerInfo ewi = new ExprWalkerInfo();
+ Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
+
+ // if this filter is generated one, predicates need not to be extracted
+ ExprWalkerInfo ewi = owi.getPrunedPreds(op);
// Don't push a sampling predicate since createFilter() always creates filter
// with isSamplePred = false. Also, the filterop with sampling pred is always
// a child of TableScan, so there is no need to push this predicate.
- if (!((FilterOperator)op).getConf().getIsSamplingPred()) {
+ if (ewi == null && !((FilterOperator)op).getConf().getIsSamplingPred()) {
// get pushdown predicates for this operator's predicate
+ ExprNodeDesc predicate = (((FilterOperator) nd).getConf()).getPredicate();
ewi = ExprWalkerProcFactory.extractPushdownPreds(owi, op, predicate);
if (!ewi.isDeterministic()) {
/* predicate is not deterministic */
@@ -964,6 +964,12 @@ public final class OpProcFactory {
}
owi.getCandidateFilterOps().clear();
}
+ // push down current ppd context to newly added filter
+ ExprWalkerInfo walkerInfo = owi.getPrunedPreds(op);
+ if (walkerInfo != null) {
+ walkerInfo.getNonFinalCandidates().clear();
+ owi.putPrunedPreds(output, walkerInfo);
+ }
return output;
}
@@ -1048,7 +1054,7 @@ public final class OpProcFactory {
tableScanDesc.setFilterExpr(decomposed.pushedPredicate);
tableScanDesc.setFilterObject(decomposed.pushedPredicateObject);
- return (ExprNodeGenericFuncDesc)decomposed.residualPredicate;
+ return decomposed.residualPredicate;
}
public static NodeProcessor getFilterProc() {
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorResponse.java Wed Jul 9 03:06:21 2014
@@ -22,7 +22,7 @@ import org.apache.hadoop.hive.metastore.
/**
* Encapsulates the basic response info returned by classes the implement the
- * <code>CommandProcessor</code> interfaace. Typically <code>errorMessage</code>
+ * <code>CommandProcessor</code> interface. Typically <code>errorMessage</code>
* and <code>SQLState</code> will only be set if the <code>responseCode</code>
* is not 0.
*/
@@ -32,23 +32,36 @@ public class CommandProcessorResponse {
private String SQLState;
private Schema resSchema;
+ private Throwable exception;
+
public CommandProcessorResponse(int responseCode) {
- this(responseCode, null, null);
+ this(responseCode, null, null, null, null);
}
public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState) {
- this(responseCode, errorMessage, SQLState, null);
+ this(responseCode, errorMessage, SQLState, null, null);
+ }
+
+ public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState, Throwable exception) {
+ this(responseCode, errorMessage, SQLState, null, exception);
}
public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState, Schema schema) {
+ this(responseCode, errorMessage, SQLState, schema, null);
+ }
+
+ public CommandProcessorResponse(int responseCode, String errorMessage, String SQLState,
+ Schema schema, Throwable exception) {
this.responseCode = responseCode;
this.errorMessage = errorMessage;
this.SQLState = SQLState;
this.resSchema = schema;
+ this.exception = exception;
}
public int getResponseCode() { return responseCode; }
public String getErrorMessage() { return errorMessage; }
public String getSQLState() { return SQLState; }
public Schema getSchema() { return resSchema; }
+ public Throwable getException() { return exception; }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java Wed Jul 9 03:06:21 2014
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.security.authorization;
+import java.util.List;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -77,12 +79,12 @@ public class AuthorizationPreEventListen
}
};
- private final ThreadLocal<HiveMetastoreAuthorizationProvider> tAuthorizer
- = new ThreadLocal<HiveMetastoreAuthorizationProvider>() {
+ private final ThreadLocal<List<HiveMetastoreAuthorizationProvider>> tAuthorizers
+ = new ThreadLocal<List<HiveMetastoreAuthorizationProvider>>() {
@Override
- protected HiveMetastoreAuthorizationProvider initialValue() {
+ protected List<HiveMetastoreAuthorizationProvider> initialValue() {
try {
- return (HiveMetastoreAuthorizationProvider) HiveUtils.getAuthorizeProviderManager(
+ return HiveUtils.getMetaStoreAuthorizeProviderManagers(
tConfig.get(), HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER, tAuthenticator.get());
} catch (HiveException he) {
throw new IllegalStateException("Authorization provider instantiation failure",he);
@@ -113,12 +115,16 @@ public class AuthorizationPreEventListen
tConfig.set(context.getHandler().getConf());
// Warning note : HMSHandler.getHiveConf() is not thread-unique, .getConf() is.
tAuthenticator.get().setConf(tConfig.get());
- tAuthorizer.get().setConf(tConfig.get());
+ for(HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()){
+ authorizer.setConf(tConfig.get());
+ }
tConfigSetOnAuths.set(true); // set so we don't repeat this initialization
}
tAuthenticator.get().setMetaStoreHandler(context.getHandler());
- tAuthorizer.get().setMetaStoreHandler(context.getHandler());
+ for(HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()){
+ authorizer.setMetaStoreHandler(context.getHandler());
+ }
switch (context.getEventType()) {
case CREATE_TABLE:
@@ -148,18 +154,34 @@ public class AuthorizationPreEventListen
case LOAD_PARTITION_DONE:
// noop for now
break;
+ case AUTHORIZATION_API_CALL:
+ authorizeAuthorizationAPICall();
default:
break;
}
}
+ private void authorizeAuthorizationAPICall() throws InvalidOperationException, MetaException {
+ for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) {
+ try {
+ authorizer.authorizeAuthorizationApiInvocation();
+ } catch (AuthorizationException e) {
+ throw invalidOperationException(e);
+ } catch (HiveException e) {
+ throw metaException(e);
+ }
+ }
+ }
+
private void authorizeCreateDatabase(PreCreateDatabaseEvent context)
throws InvalidOperationException, MetaException {
try {
- tAuthorizer.get().authorize(new Database(context.getDatabase()),
- HiveOperation.CREATEDATABASE.getInputRequiredPrivileges(),
- HiveOperation.CREATEDATABASE.getOutputRequiredPrivileges());
+ for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) {
+ authorizer.authorize(new Database(context.getDatabase()),
+ HiveOperation.CREATEDATABASE.getInputRequiredPrivileges(),
+ HiveOperation.CREATEDATABASE.getOutputRequiredPrivileges());
+ }
} catch (AuthorizationException e) {
throw invalidOperationException(e);
} catch (HiveException e) {
@@ -170,9 +192,11 @@ public class AuthorizationPreEventListen
private void authorizeDropDatabase(PreDropDatabaseEvent context)
throws InvalidOperationException, MetaException {
try {
- tAuthorizer.get().authorize(new Database(context.getDatabase()),
- HiveOperation.DROPDATABASE.getInputRequiredPrivileges(),
- HiveOperation.DROPDATABASE.getOutputRequiredPrivileges());
+ for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) {
+ authorizer.authorize(new Database(context.getDatabase()),
+ HiveOperation.DROPDATABASE.getInputRequiredPrivileges(),
+ HiveOperation.DROPDATABASE.getOutputRequiredPrivileges());
+ }
} catch (AuthorizationException e) {
throw invalidOperationException(e);
} catch (HiveException e) {
@@ -183,9 +207,12 @@ public class AuthorizationPreEventListen
private void authorizeCreateTable(PreCreateTableEvent context)
throws InvalidOperationException, MetaException {
try {
- tAuthorizer.get().authorize(new TableWrapper(context.getTable()),
- HiveOperation.CREATETABLE.getInputRequiredPrivileges(),
- HiveOperation.CREATETABLE.getOutputRequiredPrivileges());
+ org.apache.hadoop.hive.ql.metadata.Table wrappedTable = new TableWrapper(context.getTable());
+ for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) {
+ authorizer.authorize(wrappedTable,
+ HiveOperation.CREATETABLE.getInputRequiredPrivileges(),
+ HiveOperation.CREATETABLE.getOutputRequiredPrivileges());
+ }
} catch (AuthorizationException e) {
throw invalidOperationException(e);
} catch (HiveException e) {
@@ -196,9 +223,12 @@ public class AuthorizationPreEventListen
private void authorizeDropTable(PreDropTableEvent context)
throws InvalidOperationException, MetaException {
try {
- tAuthorizer.get().authorize(new TableWrapper(context.getTable()),
- HiveOperation.DROPTABLE.getInputRequiredPrivileges(),
- HiveOperation.DROPTABLE.getOutputRequiredPrivileges());
+ org.apache.hadoop.hive.ql.metadata.Table wrappedTable = new TableWrapper(context.getTable());
+ for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) {
+ authorizer.authorize(wrappedTable,
+ HiveOperation.DROPTABLE.getInputRequiredPrivileges(),
+ HiveOperation.DROPTABLE.getOutputRequiredPrivileges());
+ }
} catch (AuthorizationException e) {
throw invalidOperationException(e);
} catch (HiveException e) {
@@ -208,10 +238,14 @@ public class AuthorizationPreEventListen
private void authorizeAlterTable(PreAlterTableEvent context)
throws InvalidOperationException, MetaException {
+
try {
- tAuthorizer.get().authorize(new TableWrapper(context.getOldTable()),
- null,
- new Privilege[]{Privilege.ALTER_METADATA});
+ org.apache.hadoop.hive.ql.metadata.Table wrappedTable = new TableWrapper(context.getOldTable());
+ for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) {
+ authorizer.authorize(wrappedTable,
+ null,
+ new Privilege[]{Privilege.ALTER_METADATA});
+ }
} catch (AuthorizationException e) {
throw invalidOperationException(e);
} catch (HiveException e) {
@@ -223,9 +257,13 @@ public class AuthorizationPreEventListen
throws InvalidOperationException, MetaException {
try {
for (org.apache.hadoop.hive.metastore.api.Partition mapiPart : context.getPartitions()) {
- tAuthorizer.get().authorize(new PartitionWrapper(mapiPart, context),
- HiveOperation.ALTERTABLE_ADDPARTS.getInputRequiredPrivileges(),
- HiveOperation.ALTERTABLE_ADDPARTS.getOutputRequiredPrivileges());
+ org.apache.hadoop.hive.ql.metadata.Partition wrappedPartiton = new PartitionWrapper(
+ mapiPart, context);
+ for(HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()){
+ authorizer.authorize(wrappedPartiton,
+ HiveOperation.ALTERTABLE_ADDPARTS.getInputRequiredPrivileges(),
+ HiveOperation.ALTERTABLE_ADDPARTS.getOutputRequiredPrivileges());
+ }
}
} catch (AuthorizationException e) {
throw invalidOperationException(e);
@@ -240,9 +278,13 @@ public class AuthorizationPreEventListen
throws InvalidOperationException, MetaException {
try {
org.apache.hadoop.hive.metastore.api.Partition mapiPart = context.getPartition();
- tAuthorizer.get().authorize(new PartitionWrapper(mapiPart, context),
- HiveOperation.ALTERTABLE_DROPPARTS.getInputRequiredPrivileges(),
- HiveOperation.ALTERTABLE_DROPPARTS.getOutputRequiredPrivileges());
+ org.apache.hadoop.hive.ql.metadata.Partition wrappedPartition = new PartitionWrapper(
+ mapiPart, context);
+ for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) {
+ authorizer.authorize(wrappedPartition,
+ HiveOperation.ALTERTABLE_DROPPARTS.getInputRequiredPrivileges(),
+ HiveOperation.ALTERTABLE_DROPPARTS.getOutputRequiredPrivileges());
+ }
} catch (AuthorizationException e) {
throw invalidOperationException(e);
} catch (NoSuchObjectException e) {
@@ -256,9 +298,13 @@ public class AuthorizationPreEventListen
throws InvalidOperationException, MetaException {
try {
org.apache.hadoop.hive.metastore.api.Partition mapiPart = context.getNewPartition();
- tAuthorizer.get().authorize(new PartitionWrapper(mapiPart, context),
- null,
- new Privilege[]{Privilege.ALTER_METADATA});
+ org.apache.hadoop.hive.ql.metadata.Partition wrappedPartition = new PartitionWrapper(
+ mapiPart, context);
+ for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) {
+ authorizer.authorize(wrappedPartition,
+ null,
+ new Privilege[]{Privilege.ALTER_METADATA});
+ }
} catch (AuthorizationException e) {
throw invalidOperationException(e);
} catch (NoSuchObjectException e) {
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/DefaultHiveMetastoreAuthorizationProvider.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/DefaultHiveMetastoreAuthorizationProvider.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/DefaultHiveMetastoreAuthorizationProvider.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/DefaultHiveMetastoreAuthorizationProvider.java Wed Jul 9 03:06:21 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.securi
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.ql.metadata.AuthorizationException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
public class DefaultHiveMetastoreAuthorizationProvider extends BitSetCheckedAuthorizationProvider
@@ -35,5 +36,10 @@ public class DefaultHiveMetastoreAuthori
hive_db.setHandler(handler);
}
+ @Override
+ public void authorizeAuthorizationApiInvocation() throws HiveException, AuthorizationException {
+ // default no-op implementation
+ }
+
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveMetastoreAuthorizationProvider.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveMetastoreAuthorizationProvider.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveMetastoreAuthorizationProvider.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveMetastoreAuthorizationProvider.java Wed Jul 9 03:06:21 2014
@@ -19,6 +19,8 @@
package org.apache.hadoop.hive.ql.security.authorization;
import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.ql.metadata.AuthorizationException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
/**
* HiveMetastoreAuthorizationProvider : An extension of HiveAuthorizaytionProvider
@@ -37,4 +39,10 @@ public interface HiveMetastoreAuthorizat
*/
void setMetaStoreHandler(HMSHandler handler);
+ /**
+ * Authorize metastore authorization api call.
+ */
+ void authorizeAuthorizationApiInvocation() throws HiveException, AuthorizationException;
+
+
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java Wed Jul 9 03:06:21 2014
@@ -402,4 +402,9 @@ public class StorageBasedAuthorizationPr
return ace;
}
+ @Override
+ public void authorizeAuthorizationApiInvocation() throws HiveException, AuthorizationException {
+ // no-op - SBA does not attempt to authorize auth api call. Allow it
+ }
+
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java Wed Jul 9 03:06:21 2014
@@ -372,6 +372,20 @@ public class SQLStdHiveAccessController
public List<HivePrivilegeInfo> showPrivileges(HivePrincipal principal, HivePrivilegeObject privObj)
throws HiveAuthzPluginException {
try {
+
+ // First authorize the call
+ if (principal == null) {
+ // only the admin is allowed to list privileges for any user
+ if (!isUserAdmin()) {
+ throw new HiveAccessControlException("User : " + currentUserName + " has to specify"
+ + " a user name or role in the show grant. " + ADMIN_ONLY_MSG);
+ }
+ } else {
+ //principal is specified, authorize on it
+ if (!isUserAdmin()) {
+ ensureShowGrantAllowed(principal);
+ }
+ }
IMetaStoreClient mClient = metastoreClientFactory.getHiveMetastoreClient();
List<HivePrivilegeInfo> resPrivInfos = new ArrayList<HivePrivilegeInfo>();
String principalName = principal == null ? null : principal.getName();
@@ -396,6 +410,15 @@ public class SQLStdHiveAccessController
// result object
HiveObjectRef msObjRef = msObjPriv.getHiveObject();
+
+ if (!isSupportedObjectType(msObjRef.getObjectType())) {
+ // metastore returns object type such as global GLOBAL
+ // when no object is specified.
+ // such privileges are not applicable to this authorization mode, so
+ // ignore them
+ continue;
+ }
+
HivePrivilegeObject resPrivObj = new HivePrivilegeObject(
getPluginObjType(msObjRef.getObjectType()), msObjRef.getDbName(),
msObjRef.getObjectName());
@@ -416,6 +439,46 @@ public class SQLStdHiveAccessController
}
+ private void ensureShowGrantAllowed(HivePrincipal principal)
+ throws HiveAccessControlException, HiveAuthzPluginException {
+ // if user is not an admin user, allow the request only if the user is
+ // requesting for privileges for themselves or a role they belong to
+ switch (principal.getType()) {
+ case USER:
+ if (!principal.getName().equals(currentUserName)) {
+ throw new HiveAccessControlException("User : " + currentUserName + " is not"
+ + " allowed check privileges of another user : " + principal.getName() + ". "
+ + ADMIN_ONLY_MSG);
+ }
+ break;
+ case ROLE:
+ if (!userBelongsToRole(principal.getName())) {
+ throw new HiveAccessControlException("User : " + currentUserName + " is not"
+ + " allowed check privileges of a role it does not belong to : "
+ + principal.getName() + ". " + ADMIN_ONLY_MSG);
+ }
+ break;
+ default:
+ throw new AssertionError("Unexpected principal type " + principal.getType());
+ }
+ }
+
+ /**
+ * @param roleName
+ * @return true if roleName is the name of one of the roles (including the role hierarchy)
+ * that the user belongs to.
+ * @throws HiveAuthzPluginException
+ */
+ private boolean userBelongsToRole(String roleName) throws HiveAuthzPluginException {
+ for (HiveRoleGrant role : getRolesFromMS()) {
+ // set to one of the roles user belongs to.
+ if (role.getRoleName().equalsIgnoreCase(roleName)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private HivePrivilegeObjectType getPluginObjType(HiveObjectType objectType)
throws HiveAuthzPluginException {
switch (objectType) {
@@ -423,15 +486,21 @@ public class SQLStdHiveAccessController
return HivePrivilegeObjectType.DATABASE;
case TABLE:
return HivePrivilegeObjectType.TABLE_OR_VIEW;
- case COLUMN:
- case GLOBAL:
- case PARTITION:
- throw new HiveAuthzPluginException("Unsupported object type " + objectType);
default:
throw new AssertionError("Unexpected object type " + objectType);
}
}
+ private boolean isSupportedObjectType(HiveObjectType objectType) {
+ switch (objectType) {
+ case DATABASE:
+ case TABLE:
+ return true;
+ default:
+ return false;
+ }
+ }
+
@Override
public void setCurrentRole(String roleName) throws HiveAccessControlException,
HiveAuthzPluginException {
@@ -462,7 +531,7 @@ public class SQLStdHiveAccessController
+roleName);
}
- public List<HiveRoleGrant> getCurrentRoles() throws HiveAuthzPluginException {
+ private List<HiveRoleGrant> getCurrentRoles() throws HiveAuthzPluginException {
initUserRoles();
return currentRoles;
}
@@ -516,6 +585,11 @@ public class SQLStdHiveAccessController
public List<HiveRoleGrant> getRoleGrantInfoForPrincipal(HivePrincipal principal)
throws HiveAuthzPluginException, HiveAccessControlException {
try {
+ // first authorize the call
+ if (!isUserAdmin()) {
+ ensureShowGrantAllowed(principal);
+ }
+
List<RolePrincipalGrant> roleGrants = getRoleGrants(principal.getName(),
AuthorizationUtils.getThriftPrincipalType(principal.getType()));
List<HiveRoleGrant> hiveRoleGrants = new ArrayList<HiveRoleGrant>(roleGrants.size());
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Wed Jul 9 03:06:21 2014
@@ -374,8 +374,9 @@ public class SessionState {
HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER);
authenticator.setSessionState(this);
+ String clsStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER);
authorizer = HiveUtils.getAuthorizeProviderManager(conf,
- HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, authenticator, true);
+ clsStr, authenticator, true);
if (authorizer == null) {
// if it was null, the new authorization plugin must be specified in
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFFirstValue.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFFirstValue.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFFirstValue.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFFirstValue.java Wed Jul 9 03:06:21 2014
@@ -18,6 +18,9 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import java.util.ArrayDeque;
+import java.util.Deque;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.Description;
@@ -25,7 +28,11 @@ import org.apache.hadoop.hive.ql.exec.UD
import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec;
+import org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef;
+import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
@@ -163,7 +170,165 @@ public class GenericUDAFFirstValue exten
{
return ((FirstValueBuffer) agg).val;
}
+
+ @Override
+ public GenericUDAFEvaluator getWindowingEvaluator(WindowFrameDef wFrmDef) {
+ BoundaryDef start = wFrmDef.getStart();
+ BoundaryDef end = wFrmDef.getEnd();
+ return new FirstValStreamingFixedWindow(this, start.getAmt(),
+ end.getAmt());
+ }
}
+
+ static class ValIndexPair {
+ Object val;
+ int idx;
+
+ ValIndexPair(Object val, int idx) {
+ this.val = val;
+ this.idx = idx;
+ }
+ }
+
+ static class FirstValStreamingFixedWindow extends
+ GenericUDAFStreamingEvaluator<Object> {
+
+ class State extends GenericUDAFStreamingEvaluator<Object>.StreamingState {
+ private final Deque<ValIndexPair> valueChain;
+
+ public State(int numPreceding, int numFollowing, AggregationBuffer buf) {
+ super(numPreceding, numFollowing, buf);
+ valueChain = new ArrayDeque<ValIndexPair>(numPreceding + numFollowing
+ + 1);
+ }
+
+ @Override
+ public int estimate() {
+ if (!(wrappedBuf instanceof AbstractAggregationBuffer)) {
+ return -1;
+ }
+ int underlying = ((AbstractAggregationBuffer) wrappedBuf).estimate();
+ if (underlying == -1) {
+ return -1;
+ }
+ if (numPreceding == BoundarySpec.UNBOUNDED_AMOUNT) {
+ return -1;
+ }
+ /*
+ * sz Estimate = sz needed by underlying AggBuffer + sz for results + sz
+ * for maxChain + 3 * JavaDataModel.PRIMITIVES1 sz of results = sz of
+ * underlying * wdwSz sz of maxChain = sz of underlying * wdwSz
+ */
+
+ int wdwSz = numPreceding + numFollowing + 1;
+ return underlying + (underlying * wdwSz) + (underlying * wdwSz)
+ + (3 * JavaDataModel.PRIMITIVES1);
+ }
+
+ protected void reset() {
+ valueChain.clear();
+ super.reset();
+ }
+ }
+
+ public FirstValStreamingFixedWindow(GenericUDAFEvaluator wrappedEval,
+ int numPreceding, int numFollowing) {
+ super(wrappedEval, numPreceding, numFollowing);
+ }
+
+ @Override
+ public int getRowsRemainingAfterTerminate() throws HiveException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ AggregationBuffer underlying = wrappedEval.getNewAggregationBuffer();
+ return new State(numPreceding, numFollowing, underlying);
+ }
+
+ protected ObjectInspector inputOI() {
+ return ((GenericUDAFFirstValueEvaluator) wrappedEval).inputOI;
+ }
+
+ @Override
+ public void iterate(AggregationBuffer agg, Object[] parameters)
+ throws HiveException {
+
+ State s = (State) agg;
+ FirstValueBuffer fb = (FirstValueBuffer) s.wrappedBuf;
+
+ /*
+ * on firstRow invoke underlying evaluator to initialize skipNulls flag.
+ */
+ if (fb.firstRow) {
+ wrappedEval.iterate(fb, parameters);
+ }
+
+ Object o = ObjectInspectorUtils.copyToStandardObject(parameters[0],
+ inputOI(), ObjectInspectorCopyOption.WRITABLE);
+
+ /*
+ * add row to chain. except in case of UNB preceding: - only 1 firstVal
+ * needs to be tracked.
+ */
+ if (s.numPreceding != BoundarySpec.UNBOUNDED_AMOUNT
+ || s.valueChain.isEmpty()) {
+ /*
+ * add value to chain if it is not null or if skipNulls is false.
+ */
+ if (!fb.skipNulls || o != null) {
+ s.valueChain.add(new ValIndexPair(o, s.numRows));
+ }
+ }
+
+ if (s.numRows >= (s.numFollowing)) {
+ /*
+ * if skipNulls is true and there are no rows in valueChain => all rows
+ * in partition are null so far; so add null in o/p
+ */
+ if (fb.skipNulls && s.valueChain.size() == 0) {
+ s.results.add(null);
+ } else {
+ s.results.add(s.valueChain.getFirst().val);
+ }
+ }
+ s.numRows++;
+
+ if (s.valueChain.size() > 0) {
+ int fIdx = (Integer) s.valueChain.getFirst().idx;
+ if (s.numPreceding != BoundarySpec.UNBOUNDED_AMOUNT
+ && s.numRows > fIdx + s.numPreceding + s.numFollowing) {
+ s.valueChain.removeFirst();
+ }
+ }
+ }
+
+ @Override
+ public Object terminate(AggregationBuffer agg) throws HiveException {
+ State s = (State) agg;
+ FirstValueBuffer fb = (FirstValueBuffer) s.wrappedBuf;
+ ValIndexPair r = fb.skipNulls && s.valueChain.size() == 0 ? null
+ : s.valueChain.getFirst();
+
+ for (int i = 0; i < s.numFollowing; i++) {
+ s.results.add(r == null ? null : r.val);
+ s.numRows++;
+ if (r != null) {
+ int fIdx = (Integer) r.idx;
+ if (s.numPreceding != BoundarySpec.UNBOUNDED_AMOUNT
+ && s.numRows > fIdx + s.numPreceding + s.numFollowing
+ && !s.valueChain.isEmpty()) {
+ s.valueChain.removeFirst();
+ r = !s.valueChain.isEmpty() ? s.valueChain.getFirst() : r;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLastValue.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLastValue.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLastValue.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLastValue.java Wed Jul 9 03:06:21 2014
@@ -25,6 +25,9 @@ import org.apache.hadoop.hive.ql.exec.UD
import org.apache.hadoop.hive.ql.exec.WindowFunctionDescription;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec;
+import org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef;
+import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@@ -129,11 +132,13 @@ public class GenericUDAFLastValue extend
PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
}
}
-
- if ( !lb.skipNulls || lb.val != null )
- {
- lb.val = parameters[0];
- }
+
+ Object o = ObjectInspectorUtils.copyToStandardObject(parameters[0],
+ inputOI, ObjectInspectorCopyOption.WRITABLE);
+
+ if (!lb.skipNulls || o != null) {
+ lb.val = o;
+ }
}
@Override
@@ -154,11 +159,124 @@ public class GenericUDAFLastValue extend
public Object terminate(AggregationBuffer agg) throws HiveException
{
LastValueBuffer lb = (LastValueBuffer) agg;
- return ObjectInspectorUtils.copyToStandardObject(lb.val, inputOI,
- ObjectInspectorCopyOption.WRITABLE);
+ return lb.val;
}
+
+ @Override
+ public GenericUDAFEvaluator getWindowingEvaluator(WindowFrameDef wFrmDef) {
+ BoundaryDef start = wFrmDef.getStart();
+ BoundaryDef end = wFrmDef.getEnd();
+ return new LastValStreamingFixedWindow(this, start.getAmt(), end.getAmt());
+ }
}
+ static class LastValStreamingFixedWindow extends
+ GenericUDAFStreamingEvaluator<Object> {
+
+ class State extends GenericUDAFStreamingEvaluator<Object>.StreamingState {
+ private Object lastValue;
+ private int lastIdx;
+
+ public State(int numPreceding, int numFollowing, AggregationBuffer buf) {
+ super(numPreceding, numFollowing, buf);
+ lastValue = null;
+ lastIdx = -1;
+ }
+
+ @Override
+ public int estimate() {
+ if (!(wrappedBuf instanceof AbstractAggregationBuffer)) {
+ return -1;
+ }
+ int underlying = ((AbstractAggregationBuffer) wrappedBuf).estimate();
+ if (underlying == -1) {
+ return -1;
+ }
+ return 2 * underlying;
+ }
+
+ protected void reset() {
+ lastValue = null;
+ lastIdx = -1;
+ super.reset();
+ }
+ }
+
+ public LastValStreamingFixedWindow(GenericUDAFEvaluator wrappedEval,
+ int numPreceding, int numFollowing) {
+ super(wrappedEval, numPreceding, numFollowing);
+ }
+
+ @Override
+ public int getRowsRemainingAfterTerminate() throws HiveException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ AggregationBuffer underlying = wrappedEval.getNewAggregationBuffer();
+ return new State(numPreceding, numFollowing, underlying);
+ }
+
+ protected ObjectInspector inputOI() {
+ return ((GenericUDAFLastValueEvaluator) wrappedEval).inputOI;
+ }
+
+ @Override
+ public void iterate(AggregationBuffer agg, Object[] parameters)
+ throws HiveException {
+
+ State s = (State) agg;
+ LastValueBuffer lb = (LastValueBuffer) s.wrappedBuf;
+
+ /*
+ * on firstRow invoke underlying evaluator to initialize skipNulls flag.
+ */
+ if (lb.firstRow) {
+ wrappedEval.iterate(lb, parameters);
+ }
+
+ Object o = ObjectInspectorUtils.copyToStandardObject(parameters[0],
+ inputOI(), ObjectInspectorCopyOption.WRITABLE);
+
+ if (!lb.skipNulls || o != null) {
+ s.lastValue = o;
+ s.lastIdx = s.numRows;
+ } else if (lb.skipNulls && s.lastIdx != -1) {
+ if (s.numPreceding != BoundarySpec.UNBOUNDED_AMOUNT
+ && s.numRows > s.lastIdx + s.numPreceding + s.numFollowing) {
+ s.lastValue = null;
+ s.lastIdx = -1;
+ }
+ }
+
+ if (s.numRows >= (s.numFollowing)) {
+ s.results.add(s.lastValue);
+ }
+ s.numRows++;
+ }
+
+ @Override
+ public Object terminate(AggregationBuffer agg) throws HiveException {
+ State s = (State) agg;
+ LastValueBuffer lb = (LastValueBuffer) s.wrappedBuf;
+
+ if (lb.skipNulls && s.lastIdx != -1) {
+ if (s.numPreceding != BoundarySpec.UNBOUNDED_AMOUNT
+ && s.numRows > s.lastIdx + s.numPreceding + s.numFollowing) {
+ s.lastValue = null;
+ s.lastIdx = -1;
+ }
+ }
+
+ for (int i = 0; i < s.numFollowing; i++) {
+ s.results.add(s.lastValue);
+ }
+
+ return null;
+ }
+
+ }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java Wed Jul 9 03:06:21 2014
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.serde.serd
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
@@ -166,17 +167,21 @@ public class GenericUDFFormatNumber exte
}
double xDoubleValue = 0.0;
+ float xFloatValue = 0.0f;
int xIntValue = 0;
long xLongValue = 0L;
PrimitiveObjectInspector xObjectInspector = (PrimitiveObjectInspector)argumentOIs[0];
switch (xObjectInspector.getPrimitiveCategory()) {
case VOID:
- case FLOAT:
case DOUBLE:
xDoubleValue = ((DoubleObjectInspector) argumentOIs[0]).get(arguments[0].get());
resultText.set(numberFormat.format(xDoubleValue));
break;
+ case FLOAT:
+ xFloatValue = ((FloatObjectInspector) argumentOIs[0]).get(arguments[0].get());
+ resultText.set(numberFormat.format(xFloatValue));
+ break;
case BYTE:
case SHORT:
case INT:
Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java Wed Jul 9 03:06:21 2014
@@ -177,7 +177,7 @@ public class TestSymlinkTextInputFormat
QueryPlan plan = drv.getPlan();
MapRedTask selectTask = (MapRedTask)plan.getRootTasks().get(0);
- List<Path> inputPaths = Utilities.getInputPaths(newJob, selectTask.getWork().getMapWork(), emptyScratchDir, ctx);
+ List<Path> inputPaths = Utilities.getInputPaths(newJob, selectTask.getWork().getMapWork(), emptyScratchDir, ctx, false);
Utilities.setInputPaths(newJob, inputPaths);
Utilities.setMapRedWork(newJob, selectTask.getWork(), ctx.getMRTmpPath());
Modified: hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_insertoverwrite_nodel.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_insertoverwrite_nodel.q?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_insertoverwrite_nodel.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_insertoverwrite_nodel.q Wed Jul 9 03:06:21 2014
@@ -6,8 +6,11 @@ set user.name=hive_test_user;
-- check insert overwrite without delete priv
create table t1(i int);
grant insert on table t1 to user user1;
-show grant on table t1;
+
+show grant user hive_test_user on table t1;
set user.name=user1;
+show grant user user1 on table t1;
+
create table user1tab(i int);
insert overwrite table t1 select * from user1tab;
Modified: hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_priv_current_role_neg.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_priv_current_role_neg.q?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_priv_current_role_neg.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_priv_current_role_neg.q Wed Jul 9 03:06:21 2014
@@ -22,7 +22,10 @@ grant all on table tpriv_current_role to
set role role2;
-- switch to role2, grant should work
grant all on table tpriv_current_role to user user4;
+
+set user.name=user4;
show grant user user4 on table tpriv_current_role;
+set user.name=user2;
set role PUBLIC;
-- set role to public, should fail as role2 is not one of the current roles
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_grant_public_role.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_grant_public_role.q?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_grant_public_role.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_grant_public_role.q Wed Jul 9 03:06:21 2014
@@ -10,7 +10,8 @@ CREATE TABLE t_gpr1(i int);
GRANT ALL ON t_gpr1 TO ROLE public;
-SHOW GRANT ON TABLE t_gpr1;
+SHOW GRANT USER user1 ON TABLE t_gpr1;
+SHOW GRANT ROLE public ON TABLE t_gpr1;
set user.name=user2;
SHOW CURRENT ROLES;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_grant_table_priv.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_grant_table_priv.q?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_grant_table_priv.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_grant_table_priv.q Wed Jul 9 03:06:21 2014
@@ -10,22 +10,29 @@ CREATE TABLE table_priv1(i int);
-- grant insert privilege to another user
GRANT INSERT ON table_priv1 TO USER user2;
+
+set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv1;
+set user.name=user1;
-- grant select privilege to another user with grant
GRANT SELECT ON table_priv1 TO USER user2 with grant option;
-SHOW GRANT USER user2 ON TABLE table_priv1;
set user.name=user2;
--- change to other user - user2
+SHOW GRANT USER user2 ON TABLE table_priv1;
+
+-- changed to other user - user2
-- grant permissions to another user as user2
GRANT SELECT ON table_priv1 TO USER user3 with grant option;
-SHOW GRANT USER user3 ON TABLE table_priv1;
set user.name=user3;
+SHOW GRANT USER user3 ON TABLE table_priv1;
+
-- change to other user - user3
-- grant permissions to another user as user3
GRANT SELECT ON table_priv1 TO USER user4 with grant option;
+
+set user.name=user4;
SHOW GRANT USER user4 ON TABLE table_priv1;
set user.name=user1;
@@ -33,11 +40,12 @@ set user.name=user1;
-- grant all with grant to user22
GRANT ALL ON table_priv1 TO USER user22 with grant option;
-SHOW GRANT USER user22 ON TABLE table_priv1;
set user.name=user22;
+SHOW GRANT USER user22 ON TABLE table_priv1;
-- grant all without grant to user33
GRANT ALL ON table_priv1 TO USER user33 with grant option;
-SHOW GRANT USER user33 ON TABLE table_priv1;
+set user.name=user33;
+SHOW GRANT USER user33 ON TABLE table_priv1;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_insert.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_insert.q?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_insert.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_insert.q Wed Jul 9 03:06:21 2014
@@ -1,3 +1,4 @@
+set hive.users.in.admin.role=hive_admin_user;
set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator;
@@ -12,6 +13,9 @@ GRANT ALL ON TABLE t_select TO ROLE publ
-- grant insert privilege to another user
GRANT INSERT ON t_auth_ins TO USER userWIns;
GRANT INSERT,DELETE ON t_auth_ins TO USER userWInsAndDel;
+
+set user.name=hive_admin_user;
+set role admin;
SHOW GRANT ON TABLE t_auth_ins;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_revoke_table_priv.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_revoke_table_priv.q?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_revoke_table_priv.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_revoke_table_priv.q Wed Jul 9 03:06:21 2014
@@ -8,54 +8,88 @@ CREATE TABLE table_priv_rev(i int);
-- grant insert privilege to user2
GRANT INSERT ON table_priv_rev TO USER user2;
+
+set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv_rev;
SHOW GRANT USER user2 ON ALL;
+set user.name=user1;
-- revoke insert privilege from user2
REVOKE INSERT ON TABLE table_priv_rev FROM USER user2;
+
+set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv_rev;
+set user.name=user1;
-- grant all privileges one at a time --
-- grant insert privilege to user2
GRANT INSERT ON table_priv_rev TO USER user2;
+
+set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv_rev;
SHOW GRANT USER user2 ON ALL;
+set user.name=user1;
-- grant select privilege to user2, with grant option
GRANT SELECT ON table_priv_rev TO USER user2 WITH GRANT OPTION;
+
+set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv_rev;
+set user.name=user1;
-- grant update privilege to user2
GRANT UPDATE ON table_priv_rev TO USER user2;
+
+set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv_rev;
+set user.name=user1;
-- grant delete privilege to user2
GRANT DELETE ON table_priv_rev TO USER user2;
-SHOW GRANT USER user2 ON TABLE table_priv_rev;
+set user.name=user2;
+SHOW GRANT USER user2 ON TABLE table_priv_rev;
+set user.name=user1;
-- start revoking --
-- revoke update privilege from user2
REVOKE UPDATE ON TABLE table_priv_rev FROM USER user2;
+
+set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv_rev;
SHOW GRANT USER user2 ON ALL;
+set user.name=user1;
-- revoke DELETE privilege from user2
REVOKE DELETE ON TABLE table_priv_rev FROM USER user2;
+
+set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv_rev;
+set user.name=user1;
-- revoke insert privilege from user2
REVOKE INSERT ON TABLE table_priv_rev FROM USER user2;
+
+set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv_rev;
+set user.name=user1;
-- revoke select privilege from user2
REVOKE SELECT ON TABLE table_priv_rev FROM USER user2;
+
+set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv_rev;
SHOW GRANT USER user2 ON ALL;
+set user.name=user1;
-- grant all followed by revoke all
GRANT ALL ON table_priv_rev TO USER user2;
+
+set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv_rev;
+set user.name=user1;
REVOKE ALL ON TABLE table_priv_rev FROM USER user2;
+
+set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv_rev;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_role_grant2.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_role_grant2.q?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_role_grant2.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_role_grant2.q Wed Jul 9 03:06:21 2014
@@ -22,26 +22,43 @@ set user.name=user2;
set role src_role_WadMin;
-- grant role to another user
grant src_Role_wadmin to user user3;
+
+set user.name=user3;
show role grant user user3;
+
+set user.name=user2;
-- grant role to another role
-grant src_role_wadmin to role sRc_role2;;
-show role grant role src_Role2;;
+grant src_role_wadmin to role sRc_role2;
+set user.name=hive_admin_user;
+set role ADMIN;
+grant src_role2 to user user3;
+set user.name=user3;
+-- as user3 belings to src_role2 hierarchy, its should be able to run show grant on it
+show role grant role src_Role2;
set user.name=hive_admin_user;
set role ADMIN;
+
+
+
show principals src_ROle_wadmin;
set user.name=user2;
set role src_role_wadmin;
-- revoke user from role
revoke src_rolE_wadmin from user user3;
+
+set user.name=user3;
show role grant user user3;
+set user.name=user2;
+
-- revoke role from role
revoke src_rolE_wadmin from role sRc_role2;
-show role grant role sRc_role2;
-
set user.name=hive_admin_user;
set role ADMIN;
+
+show role grant role sRc_role2;
+
show principals src_role_wadmin;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q Wed Jul 9 03:06:21 2014
@@ -7,7 +7,7 @@ set user.name=user1;
-- Test view authorization , and 'show grant' variants
create table t1(i int, j int, k int);
-show grant on table t1;
+show grant user user1 on table t1;
-- protecting certain columns
create view vt1 as select i,k from t1;
@@ -23,7 +23,9 @@ show grant user user1 on all;
grant select on vt1 to user user2;
grant insert on table vt1 to user user3;
+set user.name=user2;
show grant user user2 on table vt1;
+set user.name=user3;
show grant user user3 on table vt1;
@@ -33,20 +35,30 @@ select * from vt1;
set user.name=user1;
grant all on table vt2 to user user2;
+
+set user.name=user2;
show grant user user2 on table vt2;
show grant user user2 on all;
+set user.name=user1;
revoke all on vt2 from user user2;
+
+set user.name=user2;
show grant user user2 on table vt2;
-show grant on table vt2;
+set user.name=hive_admin_user;
+set role admin;
+show grant on table vt2;
+set user.name=user1;
revoke select on table vt1 from user user2;
-show grant user user2 on table vt1;
+set user.name=user2;
+show grant user user2 on table vt1;
show grant user user2 on all;
+set user.name=user3;
-- grant privileges on roles for view, after next statement
show grant user user3 on table vt1;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/udf_format_number.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/udf_format_number.q?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/udf_format_number.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/udf_format_number.q Wed Jul 9 03:06:21 2014
@@ -57,3 +57,9 @@ SELECT format_number(-922337203685477580
format_number(4.9E-324, 324),
format_number(1.7976931348623157E308, 308)
FROM src tablesample (1 rows);
+
+-- floats
+SELECT format_number(CAST(12332.123456 AS FLOAT), 4),
+ format_number(CAST(12332.1 AS FLOAT), 4),
+ format_number(CAST(-12332.2 AS FLOAT), 0)
+FROM src tablesample (1 rows);
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/windowing_navfn.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/windowing_navfn.q?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/windowing_navfn.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/windowing_navfn.q Wed Jul 9 03:06:21 2014
@@ -29,3 +29,46 @@ select s, first_value(s) over (partition
select t, s, i, last_value(i) over (partition by t order by s)
from over10k where (s = 'oscar allen' or s = 'oscar carson') and t = 10;
+
+drop table if exists wtest;
+create table wtest as
+select a, b
+from
+(
+SELECT explode(
+ map(
+ 3, array(1,2,3,4,5),
+ 1, array(int(null),int(null),int(null), int(null), int(null)),
+ 2, array(1,null,2, null, 3)
+ )
+ ) as (a,barr) FROM (select * from src limit 1) s
+ ) s1 lateral view explode(barr) arr as b;
+
+select a, b,
+first_value(b) over (partition by a order by b rows between 1 preceding and 1 following ) ,
+first_value(b, true) over (partition by a order by b rows between 1 preceding and 1 following ) ,
+first_value(b) over (partition by a order by b rows between unbounded preceding and 1 following ) ,
+first_value(b, true) over (partition by a order by b rows between unbounded preceding and 1 following )
+from wtest;
+
+
+select a, b,
+first_value(b) over (partition by a order by b desc rows between 1 preceding and 1 following ) ,
+first_value(b, true) over (partition by a order by b desc rows between 1 preceding and 1 following ) ,
+first_value(b) over (partition by a order by b desc rows between unbounded preceding and 1 following ) ,
+first_value(b, true) over (partition by a order by b desc rows between unbounded preceding and 1 following )
+from wtest;
+
+select a, b,
+last_value(b) over (partition by a order by b rows between 1 preceding and 1 following ) ,
+last_value(b, true) over (partition by a order by b rows between 1 preceding and 1 following ) ,
+last_value(b) over (partition by a order by b rows between unbounded preceding and 1 following ) ,
+last_value(b, true) over (partition by a order by b rows between unbounded preceding and 1 following )
+from wtest;
+
+select a, b,
+last_value(b) over (partition by a order by b desc rows between 1 preceding and 1 following ) ,
+last_value(b, true) over (partition by a order by b desc rows between 1 preceding and 1 following ) ,
+last_value(b) over (partition by a order by b desc rows between unbounded preceding and 1 following ) ,
+last_value(b, true) over (partition by a order by b desc rows between unbounded preceding and 1 following )
+from wtest;
\ No newline at end of file
Modified: hive/branches/cbo/ql/src/test/resources/orc-file-dump-dictionary-threshold.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/resources/orc-file-dump-dictionary-threshold.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/resources/orc-file-dump-dictionary-threshold.out (original)
+++ hive/branches/cbo/ql/src/test/resources/orc-file-dump-dictionary-threshold.out Wed Jul 9 03:06:21 2014
@@ -90,15 +90,15 @@ Stripes:
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DIRECT_V2
- Stripe: offset: 1800000 data: 177935 rows: 1000 tail: 67 index: 813
- Stream: column 0 section ROW_INDEX start: 1800000 length 10
- Stream: column 1 section ROW_INDEX start: 1800010 length 36
- Stream: column 2 section ROW_INDEX start: 1800046 length 39
- Stream: column 3 section ROW_INDEX start: 1800085 length 728
- Stream: column 1 section DATA start: 1800813 length 4007
- Stream: column 2 section DATA start: 1804820 length 8007
- Stream: column 3 section DATA start: 1812827 length 164661
- Stream: column 3 section LENGTH start: 1977488 length 1260
+ Stripe: offset: 1751794 data: 177935 rows: 1000 tail: 67 index: 813
+ Stream: column 0 section ROW_INDEX start: 1751794 length 10
+ Stream: column 1 section ROW_INDEX start: 1751804 length 36
+ Stream: column 2 section ROW_INDEX start: 1751840 length 39
+ Stream: column 3 section ROW_INDEX start: 1751879 length 728
+ Stream: column 1 section DATA start: 1752607 length 4007
+ Stream: column 2 section DATA start: 1756614 length 8007
+ Stream: column 3 section DATA start: 1764621 length 164661
+ Stream: column 3 section LENGTH start: 1929282 length 1260
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Modified: hive/branches/cbo/ql/src/test/resources/orc-file-dump.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/resources/orc-file-dump.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/resources/orc-file-dump.out (original)
+++ hive/branches/cbo/ql/src/test/resources/orc-file-dump.out Wed Jul 9 03:06:21 2014
@@ -80,30 +80,30 @@ Stripes:
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DICTIONARY_V2
- Stripe: offset: 200000 data: 63796 rows: 5000 tail: 74 index: 123
- Stream: column 0 section ROW_INDEX start: 200000 length 10
- Stream: column 1 section ROW_INDEX start: 200010 length 35
- Stream: column 2 section ROW_INDEX start: 200045 length 39
- Stream: column 3 section ROW_INDEX start: 200084 length 39
- Stream: column 1 section DATA start: 200123 length 20029
- Stream: column 2 section DATA start: 220152 length 40035
- Stream: column 3 section DATA start: 260187 length 3574
- Stream: column 3 section LENGTH start: 263761 length 25
- Stream: column 3 section DICTIONARY_DATA start: 263786 length 133
+ Stripe: offset: 191881 data: 63796 rows: 5000 tail: 74 index: 123
+ Stream: column 0 section ROW_INDEX start: 191881 length 10
+ Stream: column 1 section ROW_INDEX start: 191891 length 35
+ Stream: column 2 section ROW_INDEX start: 191926 length 39
+ Stream: column 3 section ROW_INDEX start: 191965 length 39
+ Stream: column 1 section DATA start: 192004 length 20029
+ Stream: column 2 section DATA start: 212033 length 40035
+ Stream: column 3 section DATA start: 252068 length 3574
+ Stream: column 3 section LENGTH start: 255642 length 25
+ Stream: column 3 section DICTIONARY_DATA start: 255667 length 133
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Encoding column 3: DICTIONARY_V2
- Stripe: offset: 263993 data: 12940 rows: 1000 tail: 71 index: 123
- Stream: column 0 section ROW_INDEX start: 263993 length 10
- Stream: column 1 section ROW_INDEX start: 264003 length 36
- Stream: column 2 section ROW_INDEX start: 264039 length 39
- Stream: column 3 section ROW_INDEX start: 264078 length 38
- Stream: column 1 section DATA start: 264116 length 4007
- Stream: column 2 section DATA start: 268123 length 8007
- Stream: column 3 section DATA start: 276130 length 768
- Stream: column 3 section LENGTH start: 276898 length 25
- Stream: column 3 section DICTIONARY_DATA start: 276923 length 133
+ Stripe: offset: 255874 data: 12940 rows: 1000 tail: 71 index: 123
+ Stream: column 0 section ROW_INDEX start: 255874 length 10
+ Stream: column 1 section ROW_INDEX start: 255884 length 36
+ Stream: column 2 section ROW_INDEX start: 255920 length 39
+ Stream: column 3 section ROW_INDEX start: 255959 length 38
+ Stream: column 1 section DATA start: 255997 length 4007
+ Stream: column 2 section DATA start: 260004 length 8007
+ Stream: column 3 section DATA start: 268011 length 768
+ Stream: column 3 section LENGTH start: 268779 length 25
+ Stream: column 3 section DICTIONARY_DATA start: 268804 length 133
Encoding column 0: DIRECT
Encoding column 1: DIRECT_V2
Encoding column 2: DIRECT_V2
Modified: hive/branches/cbo/ql/src/test/results/clientnegative/authorization_insertoverwrite_nodel.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientnegative/authorization_insertoverwrite_nodel.q.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientnegative/authorization_insertoverwrite_nodel.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientnegative/authorization_insertoverwrite_nodel.q.out Wed Jul 9 03:06:21 2014
@@ -13,14 +13,18 @@ PREHOOK: Output: default@t1
POSTHOOK: query: grant insert on table t1 to user user1
POSTHOOK: type: GRANT_PRIVILEGE
POSTHOOK: Output: default@t1
-PREHOOK: query: show grant on table t1
+PREHOOK: query: show grant user hive_test_user on table t1
PREHOOK: type: SHOW_GRANT
-POSTHOOK: query: show grant on table t1
+POSTHOOK: query: show grant user hive_test_user on table t1
POSTHOOK: type: SHOW_GRANT
default t1 hive_test_user USER DELETE true -1 hive_test_user
default t1 hive_test_user USER INSERT true -1 hive_test_user
default t1 hive_test_user USER SELECT true -1 hive_test_user
default t1 hive_test_user USER UPDATE true -1 hive_test_user
+PREHOOK: query: show grant user user1 on table t1
+PREHOOK: type: SHOW_GRANT
+POSTHOOK: query: show grant user user1 on table t1
+POSTHOOK: type: SHOW_GRANT
default t1 user1 USER INSERT false -1 hive_test_user
PREHOOK: query: create table user1tab(i int)
PREHOOK: type: CREATETABLE
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/alter_partition_coltype.q.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
Files hive/branches/cbo/ql/src/test/results/clientpositive/alter_partition_coltype.q.out (original) and hive/branches/cbo/ql/src/test/results/clientpositive/alter_partition_coltype.q.out Wed Jul 9 03:06:21 2014 differ
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
Files hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out (original) and hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out Wed Jul 9 03:06:21 2014 differ
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_join.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_join.q.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
Files hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_join.q.out (original) and hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_join.q.out Wed Jul 9 03:06:21 2014 differ
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_select.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_select.q.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
Files hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_select.q.out (original) and hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_select.q.out Wed Jul 9 03:06:21 2014 differ
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/authorization_grant_public_role.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/authorization_grant_public_role.q.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/authorization_grant_public_role.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/authorization_grant_public_role.q.out Wed Jul 9 03:06:21 2014
@@ -19,18 +19,22 @@ POSTHOOK: query: -- all privileges shoul
GRANT ALL ON t_gpr1 TO ROLE public
POSTHOOK: type: GRANT_PRIVILEGE
POSTHOOK: Output: default@t_gpr1
-PREHOOK: query: SHOW GRANT ON TABLE t_gpr1
+PREHOOK: query: SHOW GRANT USER user1 ON TABLE t_gpr1
PREHOOK: type: SHOW_GRANT
-POSTHOOK: query: SHOW GRANT ON TABLE t_gpr1
+POSTHOOK: query: SHOW GRANT USER user1 ON TABLE t_gpr1
POSTHOOK: type: SHOW_GRANT
-default t_gpr1 public ROLE DELETE false -1 user1
-default t_gpr1 public ROLE INSERT false -1 user1
-default t_gpr1 public ROLE SELECT false -1 user1
-default t_gpr1 public ROLE UPDATE false -1 user1
default t_gpr1 user1 USER DELETE true -1 user1
default t_gpr1 user1 USER INSERT true -1 user1
default t_gpr1 user1 USER SELECT true -1 user1
default t_gpr1 user1 USER UPDATE true -1 user1
+PREHOOK: query: SHOW GRANT ROLE public ON TABLE t_gpr1
+PREHOOK: type: SHOW_GRANT
+POSTHOOK: query: SHOW GRANT ROLE public ON TABLE t_gpr1
+POSTHOOK: type: SHOW_GRANT
+default t_gpr1 public ROLE DELETE false -1 user1
+default t_gpr1 public ROLE INSERT false -1 user1
+default t_gpr1 public ROLE SELECT false -1 user1
+default t_gpr1 public ROLE UPDATE false -1 user1
PREHOOK: query: SHOW CURRENT ROLES
PREHOOK: type: SHOW_ROLES
POSTHOOK: query: SHOW CURRENT ROLES
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/authorization_grant_table_priv.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/authorization_grant_table_priv.q.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/authorization_grant_table_priv.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/authorization_grant_table_priv.q.out Wed Jul 9 03:06:21 2014
@@ -40,12 +40,12 @@ POSTHOOK: query: SHOW GRANT USER user2 O
POSTHOOK: type: SHOW_GRANT
default table_priv1 user2 USER INSERT false -1 user1
default table_priv1 user2 USER SELECT true -1 user1
-PREHOOK: query: -- change to other user - user2
+PREHOOK: query: -- changed to other user - user2
-- grant permissions to another user as user2
GRANT SELECT ON table_priv1 TO USER user3 with grant option
PREHOOK: type: GRANT_PRIVILEGE
PREHOOK: Output: default@table_priv1
-POSTHOOK: query: -- change to other user - user2
+POSTHOOK: query: -- changed to other user - user2
-- grant permissions to another user as user2
GRANT SELECT ON table_priv1 TO USER user3 with grant option
POSTHOOK: type: GRANT_PRIVILEGE
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/authorization_insert.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/authorization_insert.q.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/authorization_insert.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/authorization_insert.q.out Wed Jul 9 03:06:21 2014
@@ -36,6 +36,10 @@ PREHOOK: Output: default@t_auth_ins
POSTHOOK: query: GRANT INSERT,DELETE ON t_auth_ins TO USER userWInsAndDel
POSTHOOK: type: GRANT_PRIVILEGE
POSTHOOK: Output: default@t_auth_ins
+PREHOOK: query: set role admin
+PREHOOK: type: SHOW_ROLES
+POSTHOOK: query: set role admin
+POSTHOOK: type: SHOW_ROLES
PREHOOK: query: SHOW GRANT ON TABLE t_auth_ins
PREHOOK: type: SHOW_GRANT
POSTHOOK: query: SHOW GRANT ON TABLE t_auth_ins
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/authorization_role_grant2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/authorization_role_grant2.q.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/authorization_role_grant2.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/authorization_role_grant2.q.out Wed Jul 9 03:06:21 2014
@@ -60,9 +60,19 @@ PREHOOK: type: GRANT_ROLE
POSTHOOK: query: -- grant role to another role
grant src_role_wadmin to role sRc_role2
POSTHOOK: type: GRANT_ROLE
-PREHOOK: query: show role grant role src_Role2
+PREHOOK: query: set role ADMIN
+PREHOOK: type: SHOW_ROLES
+POSTHOOK: query: set role ADMIN
+POSTHOOK: type: SHOW_ROLES
+PREHOOK: query: grant src_role2 to user user3
+PREHOOK: type: GRANT_ROLE
+POSTHOOK: query: grant src_role2 to user user3
+POSTHOOK: type: GRANT_ROLE
+PREHOOK: query: -- as user3 belings to src_role2 hierarchy, its should be able to run show grant on it
+show role grant role src_Role2
PREHOOK: type: SHOW_ROLE_GRANT
-POSTHOOK: query: show role grant role src_Role2
+POSTHOOK: query: -- as user3 belings to src_role2 hierarchy, its should be able to run show grant on it
+show role grant role src_Role2
POSTHOOK: type: SHOW_ROLE_GRANT
role grant_option grant_time grantor
src_role_wadmin false -1 user2
@@ -94,21 +104,22 @@ POSTHOOK: query: show role grant user us
POSTHOOK: type: SHOW_ROLE_GRANT
role grant_option grant_time grantor
public false -1
+src_role2 false -1 hive_admin_user
PREHOOK: query: -- revoke role from role
revoke src_rolE_wadmin from role sRc_role2
PREHOOK: type: REVOKE_ROLE
POSTHOOK: query: -- revoke role from role
revoke src_rolE_wadmin from role sRc_role2
POSTHOOK: type: REVOKE_ROLE
+PREHOOK: query: set role ADMIN
+PREHOOK: type: SHOW_ROLES
+POSTHOOK: query: set role ADMIN
+POSTHOOK: type: SHOW_ROLES
PREHOOK: query: show role grant role sRc_role2
PREHOOK: type: SHOW_ROLE_GRANT
POSTHOOK: query: show role grant role sRc_role2
POSTHOOK: type: SHOW_ROLE_GRANT
role grant_option grant_time grantor
-PREHOOK: query: set role ADMIN
-PREHOOK: type: SHOW_ROLES
-POSTHOOK: query: set role ADMIN
-POSTHOOK: type: SHOW_ROLES
PREHOOK: query: show principals src_role_wadmin
PREHOOK: type: SHOW_ROLE_PRINCIPALS
POSTHOOK: query: show principals src_role_wadmin
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out Wed Jul 9 03:06:21 2014
@@ -9,9 +9,9 @@ create table t1(i int, j int, k int)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@t1
-PREHOOK: query: show grant on table t1
+PREHOOK: query: show grant user user1 on table t1
PREHOOK: type: SHOW_GRANT
-POSTHOOK: query: show grant on table t1
+POSTHOOK: query: show grant user user1 on table t1
POSTHOOK: type: SHOW_GRANT
default t1 user1 USER DELETE true -1 user1
default t1 user1 USER INSERT true -1 user1
@@ -122,6 +122,10 @@ PREHOOK: query: show grant user user2 on
PREHOOK: type: SHOW_GRANT
POSTHOOK: query: show grant user user2 on table vt2
POSTHOOK: type: SHOW_GRANT
+PREHOOK: query: set role admin
+PREHOOK: type: SHOW_ROLES
+POSTHOOK: query: set role admin
+POSTHOOK: type: SHOW_ROLES
PREHOOK: query: show grant on table vt2
PREHOOK: type: SHOW_GRANT
POSTHOOK: query: show grant on table vt2
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/auto_join11.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/auto_join11.q.out?rev=1608997&r1=1608996&r2=1608997&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/auto_join11.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/auto_join11.q.out Wed Jul 9 03:06:21 2014
@@ -53,12 +53,12 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((key < 100) and key is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3006 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 1002 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -69,11 +69,11 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col3
- Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col3
- Statistics: Num rows: 16 Data size: 3306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 1102 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(hash(_col0,_col3))
mode: hash