You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/07/27 01:46:00 UTC
svn commit: r1613740 [17/29] - in /hive/branches/spark: ./
beeline/src/java/org/apache/hive/beeline/ beeline/src/main/resources/
beeline/src/test/org/apache/hive/beeline/ bin/ bin/ext/ common/
common/src/java/org/apache/hadoop/hive/ant/ common/src/java...
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java Sat Jul 26 23:45:46 2014
@@ -53,6 +53,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.security.authorization.AuthorizationUtils;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException;
+import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrincipal;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilege;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
@@ -185,8 +186,12 @@ public class SQLAuthorizationUtils {
// get privileges for this user and its role on this object
PrincipalPrivilegeSet thrifPrivs = null;
try {
+ HiveObjectRef objectRef = AuthorizationUtils.getThriftHiveObjectRef(hivePrivObject);
+ if (objectRef.getObjectType() == null) {
+ objectRef.setObjectType(HiveObjectType.GLOBAL);
+ }
thrifPrivs = metastoreClient.get_privilege_set(
- AuthorizationUtils.getThriftHiveObjectRef(hivePrivObject), userName, null);
+ objectRef, userName, null);
} catch (MetaException e) {
throwGetPrivErr(e, hivePrivObject, userName);
} catch (TException e) {
@@ -259,7 +264,7 @@ public class SQLAuthorizationUtils {
Table thriftTableObj = null;
try {
thriftTableObj = metastoreClient.getTable(hivePrivObject.getDbname(),
- hivePrivObject.getTableViewURI());
+ hivePrivObject.getObjectName());
} catch (Exception e) {
throwGetObjErr(e, hivePrivObject);
}
@@ -347,18 +352,15 @@ public class SQLAuthorizationUtils {
}
}
- public static void assertNoMissingPrivilege(Collection<SQLPrivTypeGrant> missingPrivs,
- HivePrincipal hivePrincipal, HivePrivilegeObject hivePrivObject)
- throws HiveAccessControlException {
+ public static void addMissingPrivMsg(Collection<SQLPrivTypeGrant> missingPrivs,
+ HivePrivilegeObject hivePrivObject, List<String> deniedMessages) {
if (missingPrivs.size() != 0) {
// there are some required privileges missing, create error message
// sort the privileges so that error message is deterministic (for tests)
List<SQLPrivTypeGrant> sortedmissingPrivs = new ArrayList<SQLPrivTypeGrant>(missingPrivs);
Collections.sort(sortedmissingPrivs);
-
- String errMsg = "Permission denied. " + hivePrincipal
- + " does not have following privileges on " + hivePrivObject + " : " + sortedmissingPrivs;
- throw new HiveAccessControlException(errMsg.toString());
+ String errMsg = sortedmissingPrivs + " on " + hivePrivObject;
+ deniedMessages.add(errMsg);
}
}
@@ -399,5 +401,16 @@ public class SQLAuthorizationUtils {
return availPrivs;
}
+ public static void assertNoDeniedPermissions(HivePrincipal hivePrincipal,
+ HiveOperationType hiveOpType, List<String> deniedMessages) throws HiveAccessControlException {
+ if (deniedMessages.size() != 0) {
+ Collections.sort(deniedMessages);
+ String errorMessage = "Permission denied: " + hivePrincipal
+ + " does not have following privileges for operation " + hiveOpType + " "
+ + deniedMessages;
+ throw new HiveAccessControlException(errorMessage);
+ }
+ }
+
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java Sat Jul 26 23:45:46 2014
@@ -79,7 +79,7 @@ public class SQLStdHiveAccessController
private HiveRoleGrant adminRole;
private final String ADMIN_ONLY_MSG = "User has to belong to ADMIN role and "
+ "have it as current role, for this action.";
- private final String HAS_ADMIN_PRIV_MSG = "grantor need to have ADMIN privileges on role being"
+ private final String HAS_ADMIN_PRIV_MSG = "grantor need to have ADMIN OPTION on role being"
+ " granted and have it as a current role for this action.";
public static final Log LOG = LogFactory.getLog(SQLStdHiveAccessController.class);
@@ -236,7 +236,7 @@ public class SQLStdHiveAccessController
// So this will revoke privileges that are granted by other users.This is
// not SQL compliant behavior. Need to change/add a metastore api
// that has desired behavior.
- metastoreClient.revoke_privileges(new PrivilegeBag(revokePrivs));
+ metastoreClient.revoke_privileges(new PrivilegeBag(revokePrivs), grantOption);
} catch (Exception e) {
throw new HiveAuthzPluginException("Error revoking privileges", e);
}
@@ -308,11 +308,6 @@ public class SQLStdHiveAccessController
public void revokeRole(List<HivePrincipal> hivePrincipals, List<String> roleNames,
boolean grantOption, HivePrincipal grantorPrinc) throws HiveAuthzPluginException,
HiveAccessControlException {
- if (grantOption) {
- // removing grant privileges only is not supported in metastore api
- throw new HiveAuthzPluginException("Revoking only the admin privileges on "
- + "role is not currently supported");
- }
if (!(isUserAdmin() || doesUserHasAdminOption(roleNames))) {
throw new HiveAccessControlException("Current user : " + currentUserName+ " is not"
+ " allowed to revoke role. " + ADMIN_ONLY_MSG + " Otherwise, " + HAS_ADMIN_PRIV_MSG);
@@ -322,7 +317,7 @@ public class SQLStdHiveAccessController
try {
IMetaStoreClient mClient = metastoreClientFactory.getHiveMetastoreClient();
mClient.revoke_role(roleName, hivePrincipal.getName(),
- AuthorizationUtils.getThriftPrincipalType(hivePrincipal.getType()));
+ AuthorizationUtils.getThriftPrincipalType(hivePrincipal.getType()), grantOption);
} catch (Exception e) {
String msg = "Error revoking roles for " + hivePrincipal.getName() + " to role "
+ roleName + ": " + e.getMessage();
@@ -355,19 +350,24 @@ public class SQLStdHiveAccessController
+ " allowed get principals in a role. " + ADMIN_ONLY_MSG);
}
try {
- GetPrincipalsInRoleResponse princGrantInfo =
- metastoreClientFactory.getHiveMetastoreClient().get_principals_in_role(new GetPrincipalsInRoleRequest(roleName));
-
- List<HiveRoleGrant> hiveRoleGrants = new ArrayList<HiveRoleGrant>();
- for(RolePrincipalGrant thriftRoleGrant : princGrantInfo.getPrincipalGrants()){
- hiveRoleGrants.add(new HiveRoleGrant(thriftRoleGrant));
- }
- return hiveRoleGrants;
+ return getHiveRoleGrants(metastoreClientFactory.getHiveMetastoreClient(), roleName);
} catch (Exception e) {
throw new HiveAuthzPluginException("Error getting principals for all roles", e);
}
}
+ public static List<HiveRoleGrant> getHiveRoleGrants(IMetaStoreClient client, String roleName)
+ throws Exception {
+ GetPrincipalsInRoleRequest request = new GetPrincipalsInRoleRequest(roleName);
+ GetPrincipalsInRoleResponse princGrantInfo = client.get_principals_in_role(request);
+
+ List<HiveRoleGrant> hiveRoleGrants = new ArrayList<HiveRoleGrant>();
+ for(RolePrincipalGrant thriftRoleGrant : princGrantInfo.getPrincipalGrants()){
+ hiveRoleGrants.add(new HiveRoleGrant(thriftRoleGrant));
+ }
+ return hiveRoleGrants;
+ }
+
@Override
public List<HivePrivilegeInfo> showPrivileges(HivePrincipal principal, HivePrivilegeObject privObj)
throws HiveAuthzPluginException {
@@ -420,8 +420,8 @@ public class SQLStdHiveAccessController
}
HivePrivilegeObject resPrivObj = new HivePrivilegeObject(
- getPluginObjType(msObjRef.getObjectType()), msObjRef.getDbName(),
- msObjRef.getObjectName());
+ getPluginPrivilegeObjType(msObjRef.getObjectType()), msObjRef.getDbName(),
+ msObjRef.getObjectName(), msObjRef.getPartValues(), msObjRef.getColumnName());
// result grantor principal
HivePrincipal grantorPrincipal = new HivePrincipal(msGrantInfo.getGrantor(),
@@ -479,8 +479,14 @@ public class SQLStdHiveAccessController
return false;
}
- private HivePrivilegeObjectType getPluginObjType(HiveObjectType objectType)
- throws HiveAuthzPluginException {
+ /**
+ * Convert metastore object type to HivePrivilegeObjectType.
+ * Also verifies that metastore object type is of a type on which metastore privileges are
+ * supported by sql std auth.
+ * @param objectType
+ * @return corresponding HivePrivilegeObjectType
+ */
+ private HivePrivilegeObjectType getPluginPrivilegeObjType(HiveObjectType objectType) {
switch (objectType) {
case DATABASE:
return HivePrivilegeObjectType.DATABASE;
@@ -682,9 +688,6 @@ public class SQLStdHiveAccessController
LOG.debug("Configuring hooks : " + hooks);
hiveConf.setVar(ConfVars.PREEXECHOOKS, hooks);
- // set security command list to only allow set command
- hiveConf.setVar(ConfVars.HIVE_SECURITY_COMMAND_WHITELIST, "set");
-
// restrict the variables that can be set using set command to a list in whitelist
hiveConf.setIsModWhiteListEnabled(true);
String whiteListParamsStr = hiveConf.getVar(ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidator.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidator.java Sat Jul 26 23:45:46 2014
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -28,13 +29,13 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizationValidator;
+import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactory;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrincipal;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrincipal.HivePrincipalType;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType;
import org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.Operation2Privilege.IOType;
public class SQLStdHiveAuthorizationValidator implements HiveAuthorizationValidator {
@@ -57,13 +58,13 @@ public class SQLStdHiveAuthorizationVali
@Override
public void checkPrivileges(HiveOperationType hiveOpType, List<HivePrivilegeObject> inputHObjs,
- List<HivePrivilegeObject> outputHObjs) throws HiveAuthzPluginException,
- HiveAccessControlException {
+ List<HivePrivilegeObject> outputHObjs, HiveAuthzContext context)
+ throws HiveAuthzPluginException, HiveAccessControlException {
if (LOG.isDebugEnabled()) {
String msg = "Checking privileges for operation " + hiveOpType + " by user "
+ authenticator.getUserName() + " on " + " input objects " + inputHObjs
- + " and output objects " + outputHObjs;
+ + " and output objects " + outputHObjs + ". Context Info: " + context;
LOG.debug(msg);
}
@@ -71,15 +72,22 @@ public class SQLStdHiveAuthorizationVali
IMetaStoreClient metastoreClient = metastoreClientFactory.getHiveMetastoreClient();
// check privileges on input and output objects
- checkPrivileges(hiveOpType, inputHObjs, metastoreClient, userName, IOType.INPUT);
- checkPrivileges(hiveOpType, outputHObjs, metastoreClient, userName, IOType.OUTPUT);
+ List<String> deniedMessages = new ArrayList<String>();
+ checkPrivileges(hiveOpType, inputHObjs, metastoreClient, userName, IOType.INPUT, deniedMessages);
+ checkPrivileges(hiveOpType, outputHObjs, metastoreClient, userName, IOType.OUTPUT, deniedMessages);
+ SQLAuthorizationUtils.assertNoDeniedPermissions(new HivePrincipal(userName,
+ HivePrincipalType.USER), hiveOpType, deniedMessages);
}
private void checkPrivileges(HiveOperationType hiveOpType, List<HivePrivilegeObject> hiveObjects,
- IMetaStoreClient metastoreClient, String userName, IOType ioType)
+ IMetaStoreClient metastoreClient, String userName, IOType ioType, List<String> deniedMessages)
throws HiveAuthzPluginException, HiveAccessControlException {
+ if (hiveObjects == null) {
+ return;
+ }
+
// Compare required privileges and available privileges for each hive object
for (HivePrivilegeObject hiveObj : hiveObjects) {
@@ -87,26 +95,34 @@ public class SQLStdHiveAuthorizationVali
ioType);
// find available privileges
- RequiredPrivileges availPrivs;
- if (hiveObj.getType() == HivePrivilegeObjectType.LOCAL_URI
- || hiveObj.getType() == HivePrivilegeObjectType.DFS_URI) {
- availPrivs = SQLAuthorizationUtils.getPrivilegesFromFS(new Path(hiveObj.getTableViewURI()),
+ RequiredPrivileges availPrivs = new RequiredPrivileges(); //start with an empty priv set;
+ switch (hiveObj.getType()) {
+ case LOCAL_URI:
+ case DFS_URI:
+ availPrivs = SQLAuthorizationUtils.getPrivilegesFromFS(new Path(hiveObj.getObjectName()),
conf, userName);
- } else if (hiveObj.getType() == HivePrivilegeObjectType.PARTITION) {
+ break;
+ case PARTITION:
// sql std authorization is managing privileges at the table/view levels
// only
// ignore partitions
continue;
- } else {
- // get the privileges that this user has on the object
+ case COMMAND_PARAMS:
+ case FUNCTION:
+ // operations that have objects of type COMMAND_PARAMS, FUNCTION are authorized
+ // solely on the type
+ if (privController.isUserAdmin()) {
+ availPrivs.addPrivilege(SQLPrivTypeGrant.ADMIN_PRIV);
+ }
+ break;
+ default:
availPrivs = SQLAuthorizationUtils.getPrivilegesFromMetaStore(metastoreClient, userName,
hiveObj, privController.getCurrentRoleNames(), privController.isUserAdmin());
}
// Verify that there are no missing privileges
Collection<SQLPrivTypeGrant> missingPriv = requiredPrivs.findMissingPrivs(availPrivs);
- SQLAuthorizationUtils.assertNoMissingPrivilege(missingPriv, new HivePrincipal(userName,
- HivePrincipalType.USER), hiveObj);
+ SQLAuthorizationUtils.addMissingPrivMsg(missingPriv, hiveObj, deniedMessages);
}
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Sat Jul 26 23:45:46 2014
@@ -41,6 +41,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -55,6 +56,7 @@ import org.apache.hadoop.hive.ql.log.Per
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider;
import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider;
@@ -65,6 +67,8 @@ import org.apache.hadoop.hive.ql.util.Do
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.util.ReflectionUtils;
+import com.google.common.base.Preconditions;
+
/**
* SessionState encapsulates common data associated with a session.
*
@@ -75,6 +79,12 @@ import org.apache.hadoop.util.Reflection
public class SessionState {
private static final Log LOG = LogFactory.getLog(SessionState.class);
+ private static final String TMP_PREFIX = "_tmp_space.db";
+ private static final String LOCAL_SESSION_PATH_KEY = "_hive.local.session.path";
+ private static final String HDFS_SESSION_PATH_KEY = "_hive.hdfs.session.path";
+ private static final String TMP_TABLE_SPACE_KEY = "_hive.tmp_table_space";
+ private final Map<String, Map<String, Table>> tempTables = new HashMap<String, Map<String, Table>>();
+
protected ClassLoader parentLoader;
/**
@@ -163,6 +173,8 @@ public class SessionState {
private final String CONFIG_AUTHZ_SETTINGS_APPLIED_MARKER =
"hive.internal.ss.authz.settings.applied.marker";
+ private String userIpAddress;
+
/**
* Lineage state.
*/
@@ -173,6 +185,24 @@ public class SessionState {
private final String userName;
/**
+ * scratch path to use for all non-local (ie. hdfs) file system tmp folders
+ * @return Path for Scratch path for the current session
+ */
+ private Path hdfsSessionPath;
+
+ /**
+ * sub dir of hdfs session path. used to keep tmp tables
+ * @return Path for temporary tables created by the current session
+ */
+ private Path hdfsTmpTableSpace;
+
+ /**
+ * scratch directory to use for local file system tmp folders
+ * @return Path for local scratch directory for current session
+ */
+ private Path localSessionPath;
+
+ /**
* Get the lineage state stored in this session.
*
* @return LineageState
@@ -335,6 +365,7 @@ public class SessionState {
Hive.get(new HiveConf(startSs.conf)).getMSC();
ShimLoader.getHadoopShims().getUGIForConf(startSs.conf);
FileSystem.get(startSs.conf);
+ startSs.createSessionPaths(startSs.conf);
} catch (Exception e) {
// catch-all due to some exec time dependencies on session state
// that would cause ClassNoFoundException otherwise
@@ -359,6 +390,95 @@ public class SessionState {
return startSs;
}
+ public static Path getLocalSessionPath(Configuration conf) {
+ SessionState ss = SessionState.get();
+ if (ss == null) {
+ String localPathString = conf.get(LOCAL_SESSION_PATH_KEY);
+ Preconditions.checkNotNull(localPathString,
+ "Conf local session path expected to be non-null");
+ return new Path(localPathString);
+ }
+ Preconditions.checkNotNull(ss.localSessionPath,
+ "Local session path expected to be non-null");
+ return ss.localSessionPath;
+ }
+
+ public static Path getHDFSSessionPath(Configuration conf) {
+ SessionState ss = SessionState.get();
+ if (ss == null) {
+ String sessionPathString = conf.get(HDFS_SESSION_PATH_KEY);
+ Preconditions.checkNotNull(sessionPathString,
+ "Conf non-local session path expected to be non-null");
+ return new Path(sessionPathString);
+ }
+ Preconditions.checkNotNull(ss.hdfsSessionPath,
+ "Non-local session path expected to be non-null");
+ return ss.hdfsSessionPath;
+ }
+
+ public static Path getTempTableSpace(Configuration conf) {
+ SessionState ss = SessionState.get();
+ if (ss == null) {
+ String tempTablePathString = conf.get(TMP_TABLE_SPACE_KEY);
+ Preconditions.checkNotNull(tempTablePathString,
+ "Conf temp table path expected to be non-null");
+ return new Path(tempTablePathString);
+ }
+ return ss.getTempTableSpace();
+ }
+
+ public Path getTempTableSpace() {
+ Preconditions.checkNotNull(this.hdfsTmpTableSpace,
+ "Temp table path expected to be non-null");
+ return this.hdfsTmpTableSpace;
+ }
+
+ private void dropSessionPaths(Configuration conf) throws IOException {
+ if (hdfsSessionPath != null) {
+ hdfsSessionPath.getFileSystem(conf).delete(hdfsSessionPath, true);
+ }
+ if (localSessionPath != null) {
+ localSessionPath.getFileSystem(conf).delete(localSessionPath, true);
+ }
+ }
+
+ private void createSessionPaths(Configuration conf) throws IOException {
+
+ String scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION);
+ String sessionId = getSessionId();
+
+ // local & non-local tmp location is configurable. however it is the same across
+ // all external file systems
+ hdfsSessionPath =
+ new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR),
+ sessionId);
+ createPath(conf, hdfsSessionPath, scratchDirPermission);
+ conf.set(HDFS_SESSION_PATH_KEY, hdfsSessionPath.toUri().toString());
+
+ localSessionPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR),
+ sessionId);
+ createPath(conf, localSessionPath, scratchDirPermission);
+ conf.set(LOCAL_SESSION_PATH_KEY, localSessionPath.toUri().toString());
+ hdfsTmpTableSpace = new Path(hdfsSessionPath, TMP_PREFIX);
+ createPath(conf, hdfsTmpTableSpace, scratchDirPermission);
+ conf.set(TMP_TABLE_SPACE_KEY, hdfsTmpTableSpace.toUri().toString());
+ }
+
+ private void createPath(Configuration conf, Path p, String perm) throws IOException {
+ FileSystem fs = p.getFileSystem(conf);
+ p = new Path(fs.makeQualified(p).toString());
+ FsPermission fsPermission = new FsPermission(Short.parseShort(perm.trim(), 8));
+
+ if (!Utilities.createDirsWithPermission(conf, p, fsPermission)) {
+ throw new IOException("Cannot create directory: "
+ + p.toString());
+ }
+
+ // best effort to clean up if we don't shut down properly
+ fs.deleteOnExit(p);
+ }
+
+
/**
* Setup authentication and authorization plugins for this session.
*/
@@ -397,13 +517,32 @@ public class SessionState {
}
if(LOG.isDebugEnabled()){
- Object authorizationClass = getAuthorizationMode() == AuthorizationMode.V1 ?
- getAuthorizer() : getAuthorizerV2();
- LOG.debug("Session is using authorization class " + authorizationClass.getClass());
+ Object authorizationClass = getActiveAuthorizer();
+ LOG.debug("Session is using authorization class " + authorizationClass.getClass());
}
return;
}
+ public Object getActiveAuthorizer() {
+ return getAuthorizationMode() == AuthorizationMode.V1 ?
+ getAuthorizer() : getAuthorizerV2();
+ }
+
+ public Class getAuthorizerInterface() {
+ return getAuthorizationMode() == AuthorizationMode.V1 ?
+ HiveAuthorizationProvider.class : HiveAuthorizer.class;
+ }
+
+ public void setActiveAuthorizer(Object authorizer) {
+ if (authorizer instanceof HiveAuthorizationProvider) {
+ this.authorizer = (HiveAuthorizationProvider)authorizer;
+ } else if (authorizer instanceof HiveAuthorizer) {
+ this.authorizerV2 = (HiveAuthorizer) authorizer;
+ } else if (authorizer != null) {
+ throw new IllegalArgumentException("Invalid authorizer " + authorizer);
+ }
+ }
+
/**
* @param conf
* @return per-session temp file
@@ -923,6 +1062,8 @@ public class SessionState {
} finally {
tezSessionState = null;
}
+
+ dropSessionPaths(conf);
}
public AuthorizationMode getAuthorizationMode(){
@@ -992,4 +1133,24 @@ public class SessionState {
conf.set(CONFIG_AUTHZ_SETTINGS_APPLIED_MARKER, Boolean.TRUE.toString());
}
+
+ public Map<String, Map<String, Table>> getTempTables() {
+ return tempTables;
+ }
+
+ /**
+ * @return ip address for user running the query
+ */
+ public String getUserIpAddress() {
+ return userIpAddress;
+ }
+
+ /**
+ * set the ip address for user running the query
+ * @param userIpAddress
+ */
+ public void setUserIpAddress(String userIpAddress) {
+ this.userIpAddress = userIpAddress;
+ }
+
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java Sat Jul 26 23:45:46 2014
@@ -255,6 +255,10 @@ public class StatsUtils {
int avgRowSize = 0;
for (String neededCol : neededColumns) {
ColumnInfo ci = getColumnInfoForColumn(neededCol, schema);
+ if (ci == null) {
+ // No need to collect statistics of index columns
+ continue;
+ }
ObjectInspector oi = ci.getObjectInspector();
String colType = ci.getTypeName();
if (colType.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java Sat Jul 26 23:45:46 2014
@@ -594,14 +594,14 @@ public class GenericUDAFComputeStats ext
@Override
protected void updateMin(Object minValue, LongObjectInspector minFieldOI) {
- if (min == null || (minValue != null && min > minFieldOI.get(minValue))) {
+ if ((minValue != null) && (min == null || (min > minFieldOI.get(minValue)))) {
min = minFieldOI.get(minValue);
}
}
@Override
protected void updateMax(Object maxValue, LongObjectInspector maxFieldOI) {
- if (max == null || (maxValue != null && max < maxFieldOI.get(maxValue))) {
+ if ((maxValue != null ) && (max == null || (max < maxFieldOI.get(maxValue)))) {
max = maxFieldOI.get(maxValue);
}
}
@@ -657,14 +657,14 @@ public class GenericUDAFComputeStats ext
@Override
protected void updateMin(Object minValue, DoubleObjectInspector minFieldOI) {
- if (min == null || (minValue != null && min > minFieldOI.get(minValue))) {
+ if ((minValue != null) && (min == null || (min > minFieldOI.get(minValue)))) {
min = minFieldOI.get(minValue);
}
}
@Override
protected void updateMax(Object maxValue, DoubleObjectInspector maxFieldOI) {
- if (max == null || (maxValue != null && max < maxFieldOI.get(maxValue))) {
+ if ((maxValue != null ) && (max == null || (max < maxFieldOI.get(maxValue)))) {
max = maxFieldOI.get(maxValue);
}
}
@@ -995,7 +995,7 @@ public class GenericUDAFComputeStats ext
}
if (total != 0) {
- avgLength = (double)(myagg.sumLength / (1.0 * total));
+ avgLength = myagg.sumLength / (1.0 * total);
}
// Serialize the result struct
@@ -1241,7 +1241,7 @@ public class GenericUDAFComputeStats ext
long count = myagg.count + myagg.countNulls;
if (count != 0) {
- avgLength = (double)(myagg.sumLength / (1.0 * (myagg.count + myagg.countNulls)));
+ avgLength = myagg.sumLength / (1.0 * (myagg.count + myagg.countNulls));
}
// Serialize the result struct
@@ -1287,7 +1287,7 @@ public class GenericUDAFComputeStats ext
@Override
protected void updateMin(Object minValue, HiveDecimalObjectInspector minFieldOI) {
- if (min == null || (minValue != null &&
+ if ((minValue != null) && (min == null ||
min.compareTo(minFieldOI.getPrimitiveJavaObject(minValue)) > 0)) {
min = minFieldOI.getPrimitiveJavaObject(minValue);
}
@@ -1295,7 +1295,7 @@ public class GenericUDAFComputeStats ext
@Override
protected void updateMax(Object maxValue, HiveDecimalObjectInspector maxFieldOI) {
- if (max == null || (maxValue != null &&
+ if ((maxValue != null) && (max == null ||
max.compareTo(maxFieldOI.getPrimitiveJavaObject(maxValue)) < 0)) {
max = maxFieldOI.getPrimitiveJavaObject(maxValue);
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java Sat Jul 26 23:45:46 2014
@@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
import org.apache.hadoop.hive.ql.udf.UDFType;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hive.common.util.AnnotationUtils;
/**
* A Generic User-defined aggregation function (GenericUDAF) for the use with
@@ -49,7 +50,7 @@ public abstract class GenericUDAFEvaluat
public static boolean isEstimable(AggregationBuffer buffer) {
if (buffer instanceof AbstractAggregationBuffer) {
Class<? extends AggregationBuffer> clazz = buffer.getClass();
- AggregationType annotation = clazz.getAnnotation(AggregationType.class);
+ AggregationType annotation = AnnotationUtils.getAnnotation(clazz, AggregationType.class);
return annotation != null && annotation.estimable();
}
return false;
@@ -94,7 +95,7 @@ public abstract class GenericUDAFEvaluat
* Additionally setup GenericUDAFEvaluator with MapredContext before initializing.
* This is only called in runtime of MapRedTask.
*
- * @param context context
+ * @param mapredContext context
*/
public void configure(MapredContext mapredContext) {
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java Sat Jul 26 23:45:46 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf.ge
import java.text.DecimalFormat;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
@@ -32,6 +33,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.io.Text;
@@ -78,7 +80,8 @@ public class GenericUDFFormatNumber exte
+ " or \"" + serdeConstants.INT_TYPE_NAME + "\""
+ " or \"" + serdeConstants.BIGINT_TYPE_NAME + "\""
+ " or \"" + serdeConstants.DOUBLE_TYPE_NAME + "\""
- + " or \"" + serdeConstants.FLOAT_TYPE_NAME + "\", but \""
+ + " or \"" + serdeConstants.FLOAT_TYPE_NAME + "\""
+ + " or \"" + serdeConstants.DECIMAL_TYPE_NAME + "\", but \""
+ arguments[0].getTypeName() + "\" was found.");
}
@@ -104,8 +107,9 @@ public class GenericUDFFormatNumber exte
case SHORT:
case INT:
case LONG:
- case FLOAT:
case DOUBLE:
+ case FLOAT:
+ case DECIMAL:
break;
default:
throw new UDFArgumentTypeException(0, "Argument 1"
@@ -115,7 +119,8 @@ public class GenericUDFFormatNumber exte
+ " or \"" + serdeConstants.INT_TYPE_NAME + "\""
+ " or \"" + serdeConstants.BIGINT_TYPE_NAME + "\""
+ " or \"" + serdeConstants.DOUBLE_TYPE_NAME + "\""
- + " or \"" + serdeConstants.FLOAT_TYPE_NAME + "\", but \""
+ + " or \"" + serdeConstants.FLOAT_TYPE_NAME + "\""
+ + " or \"" + serdeConstants.DECIMAL_TYPE_NAME + "\", but \""
+ arguments[0].getTypeName() + "\" was found.");
}
@@ -168,6 +173,7 @@ public class GenericUDFFormatNumber exte
double xDoubleValue = 0.0;
float xFloatValue = 0.0f;
+ HiveDecimal xDecimalValue = null;
int xIntValue = 0;
long xLongValue = 0L;
@@ -182,6 +188,11 @@ public class GenericUDFFormatNumber exte
xFloatValue = ((FloatObjectInspector) argumentOIs[0]).get(arguments[0].get());
resultText.set(numberFormat.format(xFloatValue));
break;
+ case DECIMAL:
+ xDecimalValue = ((HiveDecimalObjectInspector) argumentOIs[0])
+ .getPrimitiveJavaObject(arguments[0].get());
+ resultText.set(numberFormat.format(xDecimalValue.bigDecimalValue()));
+ break;
case BYTE:
case SHORT:
case INT:
@@ -199,7 +210,8 @@ public class GenericUDFFormatNumber exte
+ " or \"" + serdeConstants.INT_TYPE_NAME + "\""
+ " or \"" + serdeConstants.BIGINT_TYPE_NAME + "\""
+ " or \"" + serdeConstants.DOUBLE_TYPE_NAME + "\""
- + " or \"" + serdeConstants.FLOAT_TYPE_NAME + "\", but \""
+ + " or \"" + serdeConstants.FLOAT_TYPE_NAME + "\""
+ + " or \"" + serdeConstants.DECIMAL_TYPE_NAME + "\", but \""
+ argumentOIs[0].getTypeName() + "\" was found.");
}
return resultText;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInFile.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInFile.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInFile.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInFile.java Sat Jul 26 23:45:46 2014
@@ -33,8 +33,8 @@ import org.apache.hadoop.hive.ql.exec.UD
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
/**
* IN_FILE(str, filename) returns true if 'str' appears in the file specified
@@ -59,29 +59,35 @@ public class GenericUDFInFile extends Ge
"IN_FILE() accepts exactly 2 arguments.");
}
- for (int i = 0; i < arguments.length; i++) {
- if (!String.class.equals(
- PrimitiveObjectInspectorUtils.
- getJavaPrimitiveClassFromObjectInspector(arguments[i]))) {
- throw new UDFArgumentTypeException(i, "The "
- + GenericUDFUtils.getOrdinal(i + 1)
- + " argument of function IN_FILE must be a string but "
- + arguments[i].toString() + " was given.");
- }
- }
-
strObjectInspector = arguments[0];
fileObjectInspector = arguments[1];
- if (!ObjectInspectorUtils.isConstantObjectInspector(fileObjectInspector)) {
- throw new UDFArgumentTypeException(1,
- "The second argument of IN_FILE() must be a constant string but " +
- fileObjectInspector.toString() + " was given.");
+ if (!isTypeCompatible(strObjectInspector)) {
+ throw new UDFArgumentTypeException(0, "The first " +
+ "argument of function IN_FILE must be a string, " +
+ "char or varchar but " +
+ strObjectInspector.toString() + " was given.");
+ }
+
+ if (((PrimitiveObjectInspector) fileObjectInspector).getPrimitiveCategory() !=
+ PrimitiveObjectInspector.PrimitiveCategory.STRING ||
+ !ObjectInspectorUtils.isConstantObjectInspector(fileObjectInspector)) {
+ throw new UDFArgumentTypeException(1, "The second " +
+ "argument of IN_FILE() must be a constant string but " +
+ fileObjectInspector.toString() + " was given.");
}
return PrimitiveObjectInspectorFactory.javaBooleanObjectInspector;
}
+ private boolean isTypeCompatible(ObjectInspector argument) {
+ PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) argument);
+ return
+ poi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.STRING ||
+ poi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.CHAR ||
+ poi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.VARCHAR;
+ }
+
@Override
public String[] getRequiredFiles() {
return new String[] {
@@ -96,12 +102,12 @@ public class GenericUDFInFile extends Ge
return null;
}
- String str = (String)ObjectInspectorUtils.copyToStandardJavaObject(
- arguments[0].get(), strObjectInspector);
+ String str = ObjectInspectorUtils.copyToStandardJavaObject(
+ arguments[0].get(), strObjectInspector).toString();
if (set == null) {
String fileName = (String)ObjectInspectorUtils.copyToStandardJavaObject(
- arguments[1].get(), fileObjectInspector);
+ arguments[1].get(), fileObjectInspector);
try {
load(new FileInputStream((new File(fileName)).getName()));
} catch (FileNotFoundException e) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToBinary.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToBinary.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToBinary.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToBinary.java Sat Jul 26 23:45:46 2014
@@ -25,13 +25,15 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.BinaryConverter;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector;
@Description(name = "binary", value = "_FUNC_(a) - cast a to binary",
- extended = "Currently only string or binary can be cast into binary")
+ extended = "Currently only string, char, varchar or binary can be cast into binary")
public class GenericUDFToBinary extends GenericUDF {
private transient PrimitiveObjectInspector argumentOI;
@@ -52,9 +54,11 @@ public class GenericUDFToBinary extends
}
if (!((argumentOI instanceof BinaryObjectInspector)
+ || (argumentOI instanceof HiveCharObjectInspector)
+ || (argumentOI instanceof HiveVarcharObjectInspector)
|| (argumentOI instanceof StringObjectInspector)
|| (argumentOI instanceof VoidObjectInspector))){
- throw new UDFArgumentException("Only string or binary data can be cast into binary " +
+ throw new UDFArgumentException("Only string, char, varchar or binary data can be cast into binary " +
"data types.");
}
baConverter = new BinaryConverter(argumentOI,
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/UDFCurrentDB.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/UDFCurrentDB.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/UDFCurrentDB.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/UDFCurrentDB.java Sat Jul 26 23:45:46 2014
@@ -54,7 +54,7 @@ public class UDFCurrentDB extends Generi
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
- throw new IllegalStateException("never");
+ return SessionState.get().getCurrentDatabase();
}
@Override
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java Sat Jul 26 23:45:46 2014
@@ -49,6 +49,7 @@ import org.apache.hadoop.hive.ql.plan.Re
import org.apache.hadoop.hive.ql.plan.TezEdgeProperty;
import org.apache.hadoop.hive.ql.plan.TezEdgeProperty.EdgeType;
import org.apache.hadoop.hive.ql.plan.TezWork;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -156,6 +157,7 @@ public class TestTezTask {
conf = new JobConf();
appLr = mock(LocalResource.class);
+ SessionState.start(new HiveConf());
session = mock(TezSession.class);
sessionState = mock(TezSessionState.class);
when(sessionState.getSession()).thenReturn(session);
@@ -166,6 +168,7 @@ public class TestTezTask {
@After
public void tearDown() throws Exception {
+ SessionState.get().close();
utils = null;
work = null;
task = null;
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestConstantVectorExpression.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestConstantVectorExpression.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestConstantVectorExpression.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestConstantVectorExpression.java Sat Jul 26 23:45:46 2014
@@ -45,28 +45,36 @@ public class TestConstantVectorExpressio
ConstantVectorExpression bytesCve = new ConstantVectorExpression(2, str.getBytes());
Decimal128 decVal = new Decimal128(25.8, (short) 1);
ConstantVectorExpression decimalCve = new ConstantVectorExpression(3, decVal);
-
+ ConstantVectorExpression nullCve = new ConstantVectorExpression(4, "string", true);
+
int size = 20;
- VectorizedRowBatch vrg = VectorizedRowGroupGenUtil.getVectorizedRowBatch(size, 4, 0);
+ VectorizedRowBatch vrg = VectorizedRowGroupGenUtil.getVectorizedRowBatch(size, 5, 0);
LongColumnVector lcv = (LongColumnVector) vrg.cols[0];
DoubleColumnVector dcv = new DoubleColumnVector(size);
BytesColumnVector bcv = new BytesColumnVector(size);
DecimalColumnVector dv = new DecimalColumnVector(5, 1);
+ BytesColumnVector bcvn = new BytesColumnVector(size);
vrg.cols[1] = dcv;
vrg.cols[2] = bcv;
vrg.cols[3] = dv;
+ vrg.cols[4] = bcvn;
longCve.evaluate(vrg);
doubleCve.evaluate(vrg);
bytesCve.evaluate(vrg);
decimalCve.evaluate(vrg);
+ nullCve.evaluate(vrg);
assertTrue(lcv.isRepeating);
assertTrue(dcv.isRepeating);
assertTrue(bcv.isRepeating);
assertEquals(17, lcv.vector[0]);
assertTrue(17.34 == dcv.vector[0]);
+ assertTrue(bcvn.isRepeating);
+ assertTrue(bcvn.isNull[0]);
+ assertTrue(!bcvn.noNulls);
+
byte[] alphaBytes = "alpha".getBytes();
assertTrue(bcv.length[0] == alphaBytes.length);
assertTrue(sameFirstKBytes(alphaBytes, bcv.vector[0], alphaBytes.length));
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorExpressionWriters.java Sat Jul 26 23:45:46 2014
@@ -25,8 +25,11 @@ import java.util.Random;
import junit.framework.Assert;
+import org.apache.hadoop.hive.common.type.Decimal128;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
@@ -36,6 +39,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
@@ -44,6 +48,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.BooleanWritable;
@@ -79,6 +84,10 @@ public class TestVectorExpressionWriters
return null;
}
+ private Writable getWritableValue(TypeInfo ti, Decimal128 value) {
+ return new HiveDecimalWritable(HiveDecimal.create(value.toBigDecimal()));
+ }
+
private Writable getWritableValue(TypeInfo ti, byte[] value) {
if (ti.equals(TypeInfoFactory.stringTypeInfo)) {
return new Text(value);
@@ -135,7 +144,7 @@ public class TestVectorExpressionWriters
VectorExpressionWriter vew = getWriter(type);
for (int i = 0; i < vectorSize; i++) {
- values[i] = vew.initValue(null);
+ values[i] = null; // setValue() should be able to handle null input
values[i] = vew.setValue(values[i], dcv, i);
if (values[i] != null) {
Writable expected = getWritableValue(type, dcv.vector[i]);
@@ -146,6 +155,41 @@ public class TestVectorExpressionWriters
}
}
+ private void testWriterDecimal(DecimalTypeInfo type) throws HiveException {
+ DecimalColumnVector dcv = VectorizedRowGroupGenUtil.generateDecimalColumnVector(type, true, false,
+ this.vectorSize, new Random(10));
+ dcv.isNull[2] = true;
+ VectorExpressionWriter vew = getWriter(type);
+ for (int i = 0; i < vectorSize; i++) {
+ Writable w = (Writable) vew.writeValue(dcv, i);
+ if (w != null) {
+ Writable expected = getWritableValue(type, dcv.vector[i]);
+ Assert.assertEquals(expected, w);
+ } else {
+ Assert.assertTrue(dcv.isNull[i]);
+ }
+ }
+ }
+
+ private void testSetterDecimal(DecimalTypeInfo type) throws HiveException {
+ DecimalColumnVector dcv = VectorizedRowGroupGenUtil.generateDecimalColumnVector(type, true, false,
+ this.vectorSize, new Random(10));
+ dcv.isNull[2] = true;
+ Object[] values = new Object[this.vectorSize];
+
+ VectorExpressionWriter vew = getWriter(type);
+ for (int i = 0; i < vectorSize; i++) {
+ values[i] = null; // setValue() should be able to handle null input
+ values[i] = vew.setValue(values[i], dcv, i);
+ if (values[i] != null) {
+ Writable expected = getWritableValue(type, dcv.vector[i]);
+ Assert.assertEquals(expected, values[i]);
+ } else {
+ Assert.assertTrue(dcv.isNull[i]);
+ }
+ }
+ }
+
private void testWriterLong(TypeInfo type) throws HiveException {
LongColumnVector lcv = VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
vectorSize, new Random(10));
@@ -178,7 +222,7 @@ public class TestVectorExpressionWriters
VectorExpressionWriter vew = getWriter(type);
for (int i = 0; i < vectorSize; i++) {
- values[i] = vew.initValue(null);
+ values[i] = null; // setValue() should be able to handle null input
values[i] = vew.setValue(values[i], lcv, i);
if (values[i] != null) {
Writable expected = getWritableValue(type, lcv.vector[i]);
@@ -290,7 +334,7 @@ public class TestVectorExpressionWriters
Object[] values = new Object[this.vectorSize];
VectorExpressionWriter vew = getWriter(type);
for (int i = 0; i < vectorSize; i++) {
- values[i] = vew.initValue(null);
+ values[i] = null; // setValue() should be able to handle null input
Writable w = (Writable) vew.setValue(values[i], bcv, i);
if (w != null) {
byte [] val = new byte[bcv.length[i]];
@@ -327,7 +371,19 @@ public class TestVectorExpressionWriters
public void testVectorExpressionWriterLong() throws HiveException {
testWriterLong(TypeInfoFactory.longTypeInfo);
}
-
+
+ @Test
+ public void testVectorExpressionWriterDecimal() throws HiveException {
+ DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(38, 18);
+ testWriterDecimal(typeInfo);
+ }
+
+ @Test
+ public void testVectorExpressionSetterDecimal() throws HiveException {
+ DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(38, 18);
+ testSetterDecimal(typeInfo);
+ }
+
@Test
public void testVectorExpressionSetterLong() throws HiveException {
testSetterLong(TypeInfoFactory.longTypeInfo);
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java Sat Jul 26 23:45:46 2014
@@ -20,9 +20,12 @@ package org.apache.hadoop.hive.ql.exec.v
import java.util.Random;
+import org.apache.hadoop.hive.common.type.Decimal128;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
public class VectorizedRowGroupGenUtil {
@@ -103,6 +106,42 @@ public class VectorizedRowGroupGenUtil {
return dcv;
}
+ public static DecimalColumnVector generateDecimalColumnVector(DecimalTypeInfo typeInfo, boolean nulls,
+ boolean repeating, int size, Random rand) {
+ DecimalColumnVector dcv =
+ new DecimalColumnVector(size, typeInfo.precision(), typeInfo.scale());
+
+ dcv.noNulls = !nulls;
+ dcv.isRepeating = repeating;
+
+ Decimal128 repeatingValue = new Decimal128();
+ do{
+ repeatingValue.update(rand.nextDouble(), (short)typeInfo.scale());
+ }while(repeatingValue.doubleValue() == 0);
+
+ int nullFrequency = generateNullFrequency(rand);
+
+ for(int i = 0; i < size; i++) {
+ if(nulls && (repeating || i % nullFrequency == 0)) {
+ dcv.isNull[i] = true;
+ dcv.vector[i] = null;//Decimal128.ONE;
+
+ }else {
+ dcv.isNull[i] = false;
+ if (repeating) {
+ dcv.vector[i].update(repeatingValue);
+ } else {
+ dcv.vector[i].update(rand.nextDouble(), (short) typeInfo.scale());
+ }
+
+ if(dcv.vector[i].doubleValue() == 0) {
+ i--;
+ }
+ }
+ }
+ return dcv;
+ }
+
private static int generateNullFrequency(Random rand) {
return 60 + rand.nextInt(20);
}
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java Sat Jul 26 23:45:46 2014
@@ -177,7 +177,7 @@ public class TestSymlinkTextInputFormat
QueryPlan plan = drv.getPlan();
MapRedTask selectTask = (MapRedTask)plan.getRootTasks().get(0);
- List<Path> inputPaths = Utilities.getInputPaths(newJob, selectTask.getWork().getMapWork(), emptyScratchDir, ctx);
+ List<Path> inputPaths = Utilities.getInputPaths(newJob, selectTask.getWork().getMapWork(), emptyScratchDir, ctx, false);
Utilities.setInputPaths(newJob, inputPaths);
Utilities.setMapRedWork(newJob, selectTask.getWork(), ctx.getMRTmpPath());
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java Sat Jul 26 23:45:46 2014
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.hooks.R
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.junit.After;
@@ -51,6 +52,7 @@ public class TestDbTxnManager {
public TestDbTxnManager() throws Exception {
TxnDbUtil.setConfValues(conf);
+ SessionState.start(conf);
ctx = new Context(conf);
LogManager.getRootLogger().setLevel(Level.DEBUG);
tearDown();
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java Sat Jul 26 23:45:46 2014
@@ -42,10 +42,10 @@ public class TestMacroSemanticAnalyzer {
@Before
public void setup() throws Exception {
conf = new HiveConf();
+ SessionState.start(conf);
context = new Context(conf);
parseDriver = new ParseDriver();
analyzer = new MacroSemanticAnalyzer(conf);
- SessionState.start(conf);
}
private ASTNode parse(String command) throws Exception {
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java Sat Jul 26 23:45:46 2014
@@ -69,10 +69,10 @@ public class TestHiveAuthorizationTaskFa
db = Mockito.mock(Hive.class);
table = new Table(DB, TABLE);
partition = new Partition(table);
+ SessionState.start(conf);
context = new Context(conf);
parseDriver = new ParseDriver();
analyzer = new DDLSemanticAnalyzer(conf, db);
- SessionState.start(conf);
Mockito.when(db.getTable(DB, TABLE, false)).thenReturn(table);
Mockito.when(db.getPartition(table, new HashMap<String, String>(), false))
.thenReturn(partition);
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveOperationType.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveOperationType.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveOperationType.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveOperationType.java Sat Jul 26 23:45:46 2014
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hive.ql.security.authorization.plugin;
-import static org.junit.Assert.*;
+import static org.junit.Assert.fail;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.junit.Test;
@@ -40,8 +40,6 @@ public class TestHiveOperationType {
fail("Unable to find corresponding type in HiveOperationType for " + op + " : " + ex );
}
}
- assertEquals("Check if HiveOperation, HiveOperationType have same number of instances",
- HiveOperation.values().length, HiveOperationType.values().length);
}
}
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessController.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessController.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessController.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessController.java Sat Jul 26 23:45:46 2014
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -42,17 +41,13 @@ public class TestSQLStdHiveAccessControl
* @throws HiveAuthzPluginException
*/
@Test
- public void checkConfigProcessing() throws HiveAuthzPluginException {
+ public void testConfigProcessing() throws HiveAuthzPluginException {
HiveConf processedConf = new HiveConf();
SQLStdHiveAccessController accessController = new SQLStdHiveAccessController(null,
processedConf, new HadoopDefaultAuthenticator());
accessController.applyAuthorizationConfigPolicy(processedConf);
- // check that unsafe commands have been disabled
- assertEquals("only set command should be allowed",
- processedConf.getVar(ConfVars.HIVE_SECURITY_COMMAND_WHITELIST), "set");
-
// check that hook to disable transforms has been added
assertTrue("Check for transform query disabling hook",
processedConf.getVar(ConfVars.PREEXECHOOKS).contains(DisallowTransformHook.class.getName()));
@@ -90,7 +85,7 @@ public class TestSQLStdHiveAccessControl
* @throws HiveAuthzPluginException
*/
@Test
- public void checkConfigProcessingCustomSetWhitelist() throws HiveAuthzPluginException {
+ public void testConfigProcessingCustomSetWhitelist() throws HiveAuthzPluginException {
HiveConf processedConf = new HiveConf();
// add custom value, including one from the default, one new one
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java Sat Jul 26 23:45:46 2014
@@ -23,7 +23,6 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -68,7 +67,7 @@ public class TestCleaner extends Compact
// Check there are no compactions requests left.
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
- Assert.assertNull(rsp.getCompacts());
+ Assert.assertEquals(0, rsp.getCompactsSize());
// Check that the files are removed
List<Path> paths = getDirectories(conf, t, null);
@@ -101,7 +100,7 @@ public class TestCleaner extends Compact
// Check there are no compactions requests left.
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
- Assert.assertNull(rsp.getCompacts());
+ Assert.assertEquals(0, rsp.getCompactsSize());
// Check that the files are removed
List<Path> paths = getDirectories(conf, t, p);
@@ -132,7 +131,7 @@ public class TestCleaner extends Compact
// Check there are no compactions requests left.
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
- Assert.assertNull(rsp.getCompacts());
+ Assert.assertEquals(0, rsp.getCompactsSize());
// Check that the files are removed
List<Path> paths = getDirectories(conf, t, null);
@@ -172,7 +171,7 @@ public class TestCleaner extends Compact
// Check there are no compactions requests left.
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
- Assert.assertNull(rsp.getCompacts());
+ Assert.assertEquals(0, rsp.getCompactsSize());
// Check that the files are removed
List<Path> paths = getDirectories(conf, t, p);
@@ -289,7 +288,7 @@ public class TestCleaner extends Compact
// Check there are no compactions requests left.
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
- Assert.assertNull(rsp.getCompacts());
+ Assert.assertEquals(0, rsp.getCompactsSize());
// Check that the files are removed
List<Path> paths = getDirectories(conf, t, p);
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java Sat Jul 26 23:45:46 2014
@@ -23,7 +23,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -187,7 +186,7 @@ public class TestInitiator extends Compa
startInitiator(conf);
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
- Assert.assertNull(rsp.getCompacts());
+ Assert.assertEquals(0, rsp.getCompactsSize());
}
@Test
@@ -246,7 +245,7 @@ public class TestInitiator extends Compa
startInitiator(conf);
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
- Assert.assertNull(rsp.getCompacts());
+ Assert.assertEquals(0, rsp.getCompactsSize());
}
@Test
@@ -379,7 +378,7 @@ public class TestInitiator extends Compa
startInitiator(conf);
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
- Assert.assertNull(rsp.getCompacts());
+ Assert.assertEquals(0, rsp.getCompactsSize());
}
@Test
@@ -492,7 +491,7 @@ public class TestInitiator extends Compa
startInitiator(conf);
ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
- Assert.assertNull(rsp.getCompacts());
+ Assert.assertEquals(0, rsp.getCompactsSize());
}
@Test
Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_addjar.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_addjar.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_addjar.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_addjar.q Sat Jul 26 23:45:46 2014
@@ -1,7 +1,4 @@
set hive.security.authorization.enabled=true;
set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory;
--- running a sql query to initialize the authorization - not needed in real HS2 mode
-show tables;
-
-add jar ${system:maven.local.repository}/org/apache/hive/hcatalog/hive-hcatalog-core/${system:hive.version}/hive-hcatalog-core-${system:hive.version}.jar;
+add jar dummy.jar
Modified: hive/branches/spark/ql/src/test/queries/clientnegative/authorization_dfs.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientnegative/authorization_dfs.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientnegative/authorization_dfs.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientnegative/authorization_dfs.q Sat Jul 26 23:45:46 2014
@@ -1,7 +1,4 @@
set hive.security.authorization.enabled=true;
set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory;
--- running a sql query to initialize the authorization - not needed in real HS2 mode
-show tables;
-dfs -ls ${system:test.tmp.dir}/
-
+dfs -ls dummy_file;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/annotate_stats_filter.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/annotate_stats_filter.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/annotate_stats_filter.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/annotate_stats_filter.q Sat Jul 26 23:45:46 2014
@@ -50,6 +50,20 @@ explain extended select * from loc_orc w
-- numRows: 0 rawDataSize: 0
explain extended select * from loc_orc where !true;
+-- Constant evaluation. true will pass all rows, false will not pass any rows
+-- numRows: 8 rawDataSize: 804
+explain extended select * from loc_orc where true;
+-- numRows: 8 rawDataSize: 804
+explain extended select * from loc_orc where 'foo';
+-- numRows: 8 rawDataSize: 804
+explain extended select * from loc_orc where true = true;
+-- numRows: 0 rawDataSize: 0
+explain extended select * from loc_orc where false = true;
+-- numRows: 0 rawDataSize: 0
+explain extended select * from loc_orc where 'foo' = 'bar';
+-- numRows: 0 rawDataSize: 0
+explain extended select * from loc_orc where false;
+
-- OR evaluation. 1 row for OH and 1 row for CA
-- numRows: 2 rawDataSize: 204
explain extended select * from loc_orc where state='OH' or state='CA';
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_9.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_9.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_9.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_9.q Sat Jul 26 23:45:46 2014
@@ -15,3 +15,11 @@ grant select on table dummy to user hive
grant select (key, value) on table dummy to user hive_test_user2;
show grant on all;
+
+revoke select on database default from user hive_test_user;
+revoke select on table dummy from user hive_test_user;
+revoke select (key, value) on table dummy from user hive_test_user;
+
+revoke select on database default from user hive_test_user2;
+revoke select on table dummy from user hive_test_user2;
+revoke select (key, value) on table dummy from user hive_test_user2;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_revoke_table_priv.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_revoke_table_priv.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_revoke_table_priv.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_revoke_table_priv.q Sat Jul 26 23:45:46 2014
@@ -74,6 +74,13 @@ set user.name=user2;
SHOW GRANT USER user2 ON TABLE table_priv_rev;
set user.name=user1;
+-- revoke grant option for select privilege from user2
+REVOKE GRANT OPTION FOR SELECT ON TABLE table_priv_rev FROM USER user2;
+
+set user.name=user2;
+SHOW GRANT USER user2 ON TABLE table_priv_rev;
+set user.name=user1;
+
-- revoke select privilege from user2
REVOKE SELECT ON TABLE table_priv_rev FROM USER user2;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_role_grant1.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_role_grant1.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_role_grant1.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_role_grant1.q Sat Jul 26 23:45:46 2014
@@ -26,6 +26,10 @@ create role src_role_wadmin;
grant src_role_wadmin to user user2 with admin option;
show role grant user user2;
+-- revoke admin option
+revoke admin option for src_role_wadmin from user user2;
+show role grant user user2;
+
-- revoke role without role keyword
revoke src_role_wadmin from user user2;
show role grant user user2;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q Sat Jul 26 23:45:46 2014
@@ -30,6 +30,8 @@ show grant user user3 on table vt1;
set user.name=user2;
+
+explain authorization select * from vt1;
select * from vt1;
set user.name=user1;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/avro_partitioned.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/avro_partitioned.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/avro_partitioned.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/avro_partitioned.q Sat Jul 26 23:45:46 2014
@@ -72,7 +72,9 @@ SELECT * FROM episodes_partitioned ORDER
SELECT * FROM episodes_partitioned WHERE doctor_pt = 6;
-- Fetch w/non-existent partition
SELECT * FROM episodes_partitioned WHERE doctor_pt = 7 LIMIT 5;
-
+-- Alter table add an empty partition
+ALTER TABLE episodes_partitioned ADD PARTITION (doctor_pt=7);
+SELECT COUNT(*) FROM episodes_partitioned;
-- Verify that reading from an Avro partition works
-- even if it has an old schema relative to the current table level schema
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/columnstats_partlvl.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/columnstats_partlvl.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/columnstats_partlvl.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/columnstats_partlvl.q Sat Jul 26 23:45:46 2014
@@ -25,3 +25,9 @@ analyze table Employee_Part partition (e
describe formatted Employee_Part.employeeID partition (employeeSalary=2000.0);
describe formatted Employee_Part.employeeName partition (employeeSalary=2000.0);
+
+explain
+analyze table Employee_Part compute statistics for columns;
+analyze table Employee_Part compute statistics for columns;
+
+describe formatted Employee_Part.employeeID;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q Sat Jul 26 23:45:46 2014
@@ -5,5 +5,6 @@ explain select cast(key as int) * 10, up
set hive.fetch.task.conversion.threshold=100;
+-- from HIVE-7397, limit + partition pruning filter
explain select * from srcpart where ds='2008-04-08' AND hr='11' limit 10;
explain select cast(key as int) * 10, upper(value) from src limit 10;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/orc_ppd_date.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/orc_ppd_date.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/orc_ppd_date.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/orc_ppd_date.q Sat Jul 26 23:45:46 2014
@@ -12,6 +12,9 @@ select sum(hash(*)) from newtypesorc whe
set hive.optimize.index.filter=true;
select sum(hash(*)) from newtypesorc where da='1970-02-20';
+set hive.optimize.index.filter=true;
+select sum(hash(*)) from newtypesorc where da= date '1970-02-20';
+
set hive.optimize.index.filter=false;
select sum(hash(*)) from newtypesorc where da=cast('1970-02-20' as date);
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/smb_mapjoin_18.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/smb_mapjoin_18.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/smb_mapjoin_18.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/smb_mapjoin_18.q Sat Jul 26 23:45:46 2014
@@ -36,8 +36,10 @@ select count(*) from test_table2 where d
select count(*) from test_table2 tablesample (bucket 1 out of 2) s where ds = '1';
select count(*) from test_table2 tablesample (bucket 2 out of 2) s where ds = '1';
+set hive.optimize.constant.propagation=false;
-- Insert data into the bucketed table by selecting from another bucketed table
-- This should be a map-only operation, one of the buckets should be empty
+
EXPLAIN
INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2')
SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' and a.key = 238;
@@ -45,6 +47,7 @@ SELECT a.key, a.value FROM test_table1 a
INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2')
SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' and a.key = 238;
+set hive.optimize.constant.propagation=true;
select count(*) from test_table2 where ds = '2';
select count(*) from test_table2 where ds = '2' and hash(key) % 2 = 0;
select count(*) from test_table2 where ds = '2' and hash(key) % 2 = 1;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/stats19.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/stats19.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/stats19.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/stats19.q Sat Jul 26 23:45:46 2014
@@ -17,7 +17,7 @@ set hive.stats.key.prefix.max.length=0;
-- The stats key should be hashed since the max length is too small
insert overwrite table stats_part partition (ds='2010-04-08', hr = '13') select key, value from src;
-set hive.stats.key.prefix.max.length=200;
+set hive.stats.key.prefix.max.length=250;
-- The stats key should not be hashed since the max length is large enough
insert overwrite table stats_part partition (ds='2010-04-08', hr = '13') select key, value from src;
@@ -41,7 +41,7 @@ insert overwrite table stats_part partit
desc formatted stats_part partition (ds='2010-04-08', hr = '13');
-set hive.stats.key.prefix.max.length=200;
+set hive.stats.key.prefix.max.length=250;
-- The stats key should not be hashed since the max length is large enough
insert overwrite table stats_part partition (ds='2010-04-08', hr = '13') select key, value from src;
@@ -66,7 +66,7 @@ set hive.stats.key.prefix.max.length=0;
-- The stats key should be hashed since the max length is too small
insert overwrite table stats_part partition (ds='2010-04-08', hr) select key, value, '13' from src;
-set hive.stats.key.prefix.max.length=200;
+set hive.stats.key.prefix.max.length=250;
-- The stats key should not be hashed since the max length is large enough
insert overwrite table stats_part partition (ds='2010-04-08', hr) select key, value, '13' from src;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/subquery_exists_having.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/subquery_exists_having.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/subquery_exists_having.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/subquery_exists_having.q Sat Jul 26 23:45:46 2014
@@ -1,4 +1,4 @@
-
+set hive.optimize.correlation=false;
-- no agg, corr
explain
@@ -22,6 +22,30 @@ having exists
)
;
+set hive.optimize.correlation=true;
+
+-- no agg, corr
+explain
+select b.key, count(*)
+from src b
+group by b.key
+having exists
+ (select a.key
+ from src a
+ where a.key = b.key and a.value > 'val_9'
+ )
+;
+
+select b.key, count(*)
+from src b
+group by b.key
+having exists
+ (select a.key
+ from src a
+ where a.key = b.key and a.value > 'val_9'
+ )
+;
+
-- view test
create view cv1 as
select b.key, count(*) as c
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/subquery_in_having.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/subquery_in_having.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/subquery_in_having.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/subquery_in_having.q Sat Jul 26 23:45:46 2014
@@ -40,6 +40,8 @@ group by key, value
having count(*) in (select count(*) from src s1 where s1.key > '9' and s1.value = b.value group by s1.key )
;
+set hive.optimize.correlation=false;
+
-- agg, non corr
explain
select p_mfgr, avg(p_size)
@@ -53,6 +55,21 @@ having b.p_mfgr in
)
;
+set hive.optimize.correlation=true;
+
+-- agg, non corr
+explain
+select p_mfgr, avg(p_size)
+from part b
+group by b.p_mfgr
+having b.p_mfgr in
+ (select p_mfgr
+ from part
+ group by p_mfgr
+ having max(p_size) - min(p_size) < 20
+ )
+;
+
-- join on agg
select b.key, min(b.value)
from src b
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/subquery_views.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/subquery_views.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/subquery_views.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/subquery_views.q Sat Jul 26 23:45:46 2014
@@ -26,6 +26,11 @@ where b.key not in
)
;
+explain
+select *
+from cv2 where cv2.key in (select key from cv2 c where c.key < '11');
+;
+
select *
from cv2 where cv2.key in (select key from cv2 c where c.key < '11');
;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/truncate_table.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/truncate_table.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/truncate_table.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/truncate_table.q Sat Jul 26 23:45:46 2014
@@ -12,24 +12,31 @@ load data local inpath '../../data/files
load data local inpath '../../data/files/kv1.txt' into table srcpart_truncate partition (ds='2008-04-09', hr='11');
load data local inpath '../../data/files/kv1.txt' into table srcpart_truncate partition (ds='2008-04-09', hr='12');
+analyze table src_truncate compute statistics;
+analyze table srcpart_truncate partition(ds,hr) compute statistics;
set hive.fetch.task.conversion=more;
+set hive.compute.query.using.stats=true;
-- truncate non-partitioned table
explain TRUNCATE TABLE src_truncate;
TRUNCATE TABLE src_truncate;
select * from src_truncate;
+select count (*) from src_truncate;
-- truncate a partition
explain TRUNCATE TABLE srcpart_truncate partition (ds='2008-04-08', hr='11');
TRUNCATE TABLE srcpart_truncate partition (ds='2008-04-08', hr='11');
select * from srcpart_truncate where ds='2008-04-08' and hr='11';
+select count(*) from srcpart_truncate where ds='2008-04-08' and hr='11';
-- truncate partitions with partial spec
explain TRUNCATE TABLE srcpart_truncate partition (ds, hr='12');
TRUNCATE TABLE srcpart_truncate partition (ds, hr='12');
select * from srcpart_truncate where hr='12';
+select count(*) from srcpart_truncate where hr='12';
-- truncate partitioned table
explain TRUNCATE TABLE srcpart_truncate;
TRUNCATE TABLE srcpart_truncate;
select * from srcpart_truncate;
+select count(*) from srcpart_truncate;
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/udf_format_number.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/udf_format_number.q?rev=1613740&r1=1613739&r2=1613740&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/udf_format_number.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/udf_format_number.q Sat Jul 26 23:45:46 2014
@@ -63,3 +63,11 @@ SELECT format_number(CAST(12332.123456 A
format_number(CAST(12332.1 AS FLOAT), 4),
format_number(CAST(-12332.2 AS FLOAT), 0)
FROM src tablesample (1 rows);
+
+-- decimals
+SELECT format_number(12332.123456BD, 4),
+ format_number(12332.123456BD, 2),
+ format_number(12332.1BD, 4),
+ format_number(-12332.2BD, 0),
+ format_number(CAST(12332.567 AS DECIMAL(8, 1)), 4)
+FROM src tablesample (1 rows);