You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/08/13 04:28:58 UTC
svn commit: r1617652 [2/7] - in /hive/branches/cbo: ./ ant/
ant/src/org/apache/hadoop/hive/ant/ beeline/
beeline/src/java/org/apache/hive/beeline/ beeline/src/main/resources/
cli/src/java/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hi...
Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java Wed Aug 13 02:28:54 2014
@@ -29,6 +29,8 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -80,11 +82,11 @@ public class HiveAlterHandler implements
FileSystem destFs = null;
boolean success = false;
- String oldTblLoc = null;
- String newTblLoc = null;
boolean moveData = false;
boolean rename = false;
Table oldt = null;
+ List<ObjectPair<Partition, String>> altps = new ArrayList<ObjectPair<Partition, String>>();
+
try {
msdb.openTransaction();
name = name.toLowerCase();
@@ -131,32 +133,32 @@ public class HiveAlterHandler implements
// if this alter is a rename, the table is not a virtual view, the user
// didn't change the default location (or new location is empty), and
- // table is not an external table, that means useris asking metastore to
+ // table is not an external table, that means user is asking metastore to
// move data to the new location corresponding to the new name
if (rename
&& !oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString())
&& (oldt.getSd().getLocation().compareTo(newt.getSd().getLocation()) == 0
|| StringUtils.isEmpty(newt.getSd().getLocation()))
&& !MetaStoreUtils.isExternalTable(oldt)) {
+
+ srcPath = new Path(oldt.getSd().getLocation());
+ srcFs = wh.getFs(srcPath);
+
// that means user is asking metastore to move data to new location
// corresponding to the new name
// get new location
- newTblLoc = wh.getTablePath(msdb.getDatabase(newt.getDbName()),
- newt.getTableName()).toString();
- Path newTblPath = constructRenamedPath(new Path(newTblLoc),
- new Path(newt.getSd().getLocation()));
- newTblLoc = newTblPath.toString();
- newt.getSd().setLocation(newTblLoc);
- oldTblLoc = oldt.getSd().getLocation();
+ Path databasePath = constructRenamedPath(
+ wh.getDefaultDatabasePath(newt.getDbName()), srcPath);
+ destPath = new Path(databasePath, newt.getTableName());
+ destFs = wh.getFs(destPath);
+
+ newt.getSd().setLocation(destPath.toString());
moveData = true;
+
// check that destination does not exist otherwise we will be
// overwriting data
- srcPath = new Path(oldTblLoc);
- srcFs = wh.getFs(srcPath);
- destPath = new Path(newTblLoc);
- destFs = wh.getFs(destPath);
// check that src and dest are on the same file system
- if (! equalsFileSystem(srcFs, destFs)) {
+ if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
throw new InvalidOperationException("table new location " + destPath
+ " is on a different file system than the old location "
+ srcPath + ". This operation is not supported");
@@ -176,22 +178,18 @@ public class HiveAlterHandler implements
+ destPath + " for table " + newt.getDbName() + "."
+ newt.getTableName());
}
+ String oldTblLocPath = srcPath.toUri().getPath();
+ String newTblLocPath = destPath.toUri().getPath();
+
// also the location field in partition
List<Partition> parts = msdb.getPartitions(dbname, name, -1);
for (Partition part : parts) {
String oldPartLoc = part.getSd().getLocation();
- Path oldPartLocPath = new Path(oldPartLoc);
- String oldTblLocPath = new Path(oldTblLoc).toUri().getPath();
- String newTblLocPath = new Path(newTblLoc).toUri().getPath();
if (oldPartLoc.contains(oldTblLocPath)) {
- Path newPartLocPath = null;
- URI oldUri = oldPartLocPath.toUri();
- String newPath = oldUri.getPath().replace(oldTblLocPath,
- newTblLocPath);
-
- newPartLocPath = new Path(oldUri.getScheme(),
- oldUri.getAuthority(),
- newPath);
+ URI oldUri = new Path(oldPartLoc).toUri();
+ String newPath = oldUri.getPath().replace(oldTblLocPath, newTblLocPath);
+ Path newPartLocPath = new Path(oldUri.getScheme(), oldUri.getAuthority(), newPath);
+ altps.add(ObjectPair.create(part, part.getSd().getLocation()));
part.getSd().setLocation(newPartLocPath.toString());
msdb.alterPartition(dbname, name, part.getValues(), part);
}
@@ -234,9 +232,23 @@ public class HiveAlterHandler implements
try {
msdb.openTransaction();
msdb.alterTable(dbname, newt.getTableName(), oldt);
+ for (ObjectPair<Partition, String> pair : altps) {
+ Partition part = pair.getFirst();
+ part.getSd().setLocation(pair.getSecond());
+ msdb.alterPartition(dbname, name, part.getValues(), part);
+ }
revertMetaDataTransaction = msdb.commitTransaction();
} catch (Exception e1) {
- LOG.error("Reverting metadata opeation failed During HDFS operation failed", e1);
+ // we should log this for manual rollback by administrator
+ LOG.error("Reverting metadata by HDFS operation failure failed During HDFS operation failed", e1);
+ LOG.error("Table " + Warehouse.getQualifiedName(newt) +
+ " should be renamed to " + Warehouse.getQualifiedName(oldt));
+ LOG.error("Table " + Warehouse.getQualifiedName(newt) +
+ " should have path " + srcPath);
+ for (ObjectPair<Partition, String> pair : altps) {
+ LOG.error("Partition " + Warehouse.getQualifiedName(pair.getFirst()) +
+ " should have path " + pair.getSecond());
+ }
if (!revertMetaDataTransaction) {
msdb.rollbackTransaction();
}
@@ -251,21 +263,6 @@ public class HiveAlterHandler implements
}
}
- /**
- * @param fs1
- * @param fs2
- * @return return true if both file system arguments point to same file system
- */
- private boolean equalsFileSystem(FileSystem fs1, FileSystem fs2) {
- //When file system cache is disabled, you get different FileSystem objects
- // for same file system, so '==' can't be used in such cases
- //FileSystem api doesn't have a .equals() function implemented, so using
- //the uri for comparison. FileSystem already uses uri+Configuration for
- //equality in its CACHE .
- //Once equality has been added in HDFS-4321, we should make use of it
- return fs1.getUri().equals(fs2.getUri());
- }
-
public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
final String name, final List<String> part_vals, final Partition new_part)
throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,
Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Wed Aug 13 02:28:54 2014
@@ -424,7 +424,7 @@ public class HiveMetaStore extends Thrif
String partitionValidationRegex =
hiveConf.getVar(HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN);
- if (partitionValidationRegex != null && partitionValidationRegex != "") {
+ if (partitionValidationRegex != null && !partitionValidationRegex.isEmpty()) {
partitionValidationPattern = Pattern.compile(partitionValidationRegex);
} else {
partitionValidationPattern = null;
@@ -3269,6 +3269,9 @@ public class HiveMetaStore extends Thrif
boolean success = false, indexTableCreated = false;
+ String[] qualified =
+ MetaStoreUtils.getQualifiedName(index.getDbName(), index.getIndexTableName());
+
try {
ms.openTransaction();
Index old_index = null;
@@ -3291,7 +3294,7 @@ public class HiveMetaStore extends Thrif
Table indexTbl = indexTable;
if (indexTbl != null) {
try {
- indexTbl = ms.getTable(index.getDbName(), index.getIndexTableName());
+ indexTbl = ms.getTable(qualified[0], qualified[1]);
} catch (Exception e) {
}
if (indexTbl != null) {
@@ -3312,7 +3315,7 @@ public class HiveMetaStore extends Thrif
if (!success) {
if (indexTableCreated) {
try {
- this.drop_table(index.getDbName(), index.getIndexTableName(), false);
+ drop_table(qualified[0], qualified[1], false);
} catch (Exception e) {
}
}
@@ -3366,8 +3369,8 @@ public class HiveMetaStore extends Thrif
String idxTblName = index.getIndexTableName();
if (idxTblName != null) {
- Table tbl = null;
- tbl = this.get_table(dbName, idxTblName);
+ String[] qualified = MetaStoreUtils.getQualifiedName(index.getDbName(), idxTblName);
+ Table tbl = get_table(qualified[0], qualified[1]);
if (tbl.getSd() == null) {
throw new MetaException("Table metadata is corrupted");
}
Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Wed Aug 13 02:28:54 2014
@@ -1538,4 +1538,12 @@ public class MetaStoreUtils {
return part.getValues().size();
}
}
+
+ public static String[] getQualifiedName(String defaultDbName, String tableName) {
+ String[] names = tableName.split("\\.");
+ if (names.length == 1) {
+ return new String[] { defaultDbName, tableName};
+ }
+ return new String[] {names[0], names[1]};
+ }
}
Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Wed Aug 13 02:28:54 2014
@@ -2847,7 +2847,8 @@ public class ObjectStore implements RawS
"Original table does not exist for the given index.");
}
- MTable indexTable = getMTable(index.getDbName(), index.getIndexTableName());
+ String[] qualified = MetaStoreUtils.getQualifiedName(index.getDbName(), index.getIndexTableName());
+ MTable indexTable = getMTable(qualified[0], qualified[1]);
if (indexTable == null) {
throw new InvalidObjectException(
"Underlying index table does not exist for the given index.");
Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java Wed Aug 13 02:28:54 2014
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.shims.ShimLoader;
@@ -186,6 +187,14 @@ public class Warehouse {
return getDnsPath(new Path(getDatabasePath(db), tableName.toLowerCase()));
}
+ public static String getQualifiedName(Table table) {
+ return table.getDbName() + "." + table.getTableName();
+ }
+
+ public static String getQualifiedName(Partition partition) {
+ return partition.getDbName() + "." + partition.getTableName() + partition.getValues();
+ }
+
public boolean mkdirs(Path f, boolean inheritPermCandidate) throws MetaException {
boolean inheritPerms = HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS) && inheritPermCandidate;
Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java Wed Aug 13 02:28:54 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.metastore.txn;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.shims.ShimLoader;
import java.sql.Connection;
import java.sql.Driver;
@@ -201,7 +202,8 @@ public class TxnDbUtil {
Properties prop = new Properties();
String driverUrl = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORECONNECTURLKEY);
String user = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME);
- String passwd = HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREPWD);
+ String passwd = ShimLoader.getHadoopShims().getPassword(conf,
+ HiveConf.ConfVars.METASTOREPWD.varname);
prop.put("user", user);
prop.put("password", passwd);
return driver.connect(driverUrl, prop);
Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java Wed Aug 13 02:28:54 2014
@@ -32,9 +32,12 @@ import org.apache.hadoop.hive.common.Val
import org.apache.hadoop.hive.common.ValidTxnListImpl;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.util.StringUtils;
import javax.sql.DataSource;
+
+import java.io.IOException;
import java.sql.*;
import java.util.*;
@@ -1602,7 +1605,13 @@ public class TxnHandler {
String driverUrl = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORECONNECTURLKEY);
String user = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME);
- String passwd = HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREPWD);
+ String passwd;
+ try {
+ passwd = ShimLoader.getHadoopShims().getPassword(conf,
+ HiveConf.ConfVars.METASTOREPWD.varname);
+ } catch (IOException err) {
+ throw new SQLException("Error getting metastore password", err);
+ }
String connectionPooler = HiveConf.getVar(conf,
HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE).toLowerCase();
Modified: hive/branches/cbo/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/cbo/pom.xml?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/pom.xml (original)
+++ hive/branches/cbo/pom.xml Wed Aug 13 02:28:54 2014
@@ -146,6 +146,7 @@
<stax.version>1.0.1</stax.version>
<slf4j.version>1.7.5</slf4j.version>
<ST4.version>4.0.4</ST4.version>
+ <super-csv.version>2.2.0</super-csv.version>
<tez.version>0.4.0-incubating</tez.version>
<tempus-fugit.version>1.1</tempus-fugit.version>
<snappy.version>0.2</snappy.version>
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Wed Aug 13 02:28:54 2014
@@ -507,7 +507,7 @@ public class Driver implements CommandPr
// get mapping of tables to columns used
ColumnAccessInfo colAccessInfo = sem.getColumnAccessInfo();
// colAccessInfo is set only in case of SemanticAnalyzer
- Map<String, Set<String>> tab2Cols = colAccessInfo != null ? colAccessInfo
+ Map<String, List<String>> tab2Cols = colAccessInfo != null ? colAccessInfo
.getTableToColumnAccessMap() : null;
doAuthorizationV2(ss, op, inputs, outputs, command, tab2Cols);
return;
@@ -608,7 +608,7 @@ public class Driver implements CommandPr
Partition partition = read.getPartition();
tbl = partition.getTable();
// use partition level authorization
- if (tableUsePartLevelAuth.get(tbl.getTableName()) == Boolean.TRUE) {
+ if (Boolean.TRUE.equals(tableUsePartLevelAuth.get(tbl.getTableName()))) {
List<String> cols = part2Cols.get(partition);
if (cols != null && cols.size() > 0) {
authorizer.authorize(partition.getTable(),
@@ -626,7 +626,7 @@ public class Driver implements CommandPr
// check, and the table authorization may already happened because of other
// partitions
if (tbl != null && !tableAuthChecked.contains(tbl.getTableName()) &&
- !(tableUsePartLevelAuth.get(tbl.getTableName()) == Boolean.TRUE)) {
+ !(Boolean.TRUE.equals(tableUsePartLevelAuth.get(tbl.getTableName())))) {
List<String> cols = tab2Cols.get(tbl);
if (cols != null && cols.size() > 0) {
authorizer.authorize(tbl, null, cols,
@@ -671,7 +671,7 @@ public class Driver implements CommandPr
//or non-existent tho such sources may still be referenced by the TableScanOperator
//if it's null then the partition probably doesn't exist so let's use table permission
if (tbl.isPartitioned() &&
- tableUsePartLevelAuth.get(tbl.getTableName()) == Boolean.TRUE) {
+ Boolean.TRUE.equals(tableUsePartLevelAuth.get(tbl.getTableName()))) {
String alias_id = topOpMap.getKey();
PrunedPartitionList partsList = PartitionPruner.prune(tableScanOp,
@@ -700,7 +700,7 @@ public class Driver implements CommandPr
}
private static void doAuthorizationV2(SessionState ss, HiveOperation op, HashSet<ReadEntity> inputs,
- HashSet<WriteEntity> outputs, String command, Map<String, Set<String>> tab2cols) throws HiveException {
+ HashSet<WriteEntity> outputs, String command, Map<String, List<String>> tab2cols) throws HiveException {
HiveAuthzContext.Builder authzContextBuilder = new HiveAuthzContext.Builder();
@@ -711,36 +711,14 @@ public class Driver implements CommandPr
authzContextBuilder.setCommandString(command);
HiveOperationType hiveOpType = getHiveOperationType(op);
- List<HivePrivilegeObject> inputsHObjs = getHivePrivObjects(inputs);
- updateInputColumnInfo(inputsHObjs, tab2cols);
+ List<HivePrivilegeObject> inputsHObjs = getHivePrivObjects(inputs, tab2cols);
+ List<HivePrivilegeObject> outputHObjs = getHivePrivObjects(outputs, null);
- List<HivePrivilegeObject> outputHObjs = getHivePrivObjects(outputs);
ss.getAuthorizerV2().checkPrivileges(hiveOpType, inputsHObjs, outputHObjs, authzContextBuilder.build());
- return;
}
- /**
- * Add column information for input table objects
- * @param inputsHObjs input HivePrivilegeObject
- * @param map table to used input columns mapping
- */
- private static void updateInputColumnInfo(List<HivePrivilegeObject> inputsHObjs,
- Map<String, Set<String>> tableName2Cols) {
- if(tableName2Cols == null) {
- return;
- }
- for(HivePrivilegeObject inputObj : inputsHObjs){
- if(inputObj.getType() != HivePrivilegeObjectType.TABLE_OR_VIEW){
- // input columns are relevant only for tables or views
- continue;
- }
- Set<String> cols = tableName2Cols.get(Table.getCompleteName(inputObj.getDbname(),
- inputObj.getObjectName()));
- inputObj.setColumns(cols);
- }
- }
-
- private static List<HivePrivilegeObject> getHivePrivObjects(HashSet<? extends Entity> privObjects) {
+ private static List<HivePrivilegeObject> getHivePrivObjects(
+ HashSet<? extends Entity> privObjects, Map<String, List<String>> tableName2Cols) {
List<HivePrivilegeObject> hivePrivobjs = new ArrayList<HivePrivilegeObject>();
if(privObjects == null){
return hivePrivobjs;
@@ -764,13 +742,17 @@ public class Driver implements CommandPr
//support for authorization on partitions needs to be added
String dbname = null;
String objName = null;
+ List<String> partKeys = null;
+ List<String> columns = null;
switch(privObject.getType()){
case DATABASE:
- dbname = privObject.getDatabase() == null ? null : privObject.getDatabase().getName();
+ dbname = privObject.getDatabase().getName();
break;
case TABLE:
- dbname = privObject.getTable() == null ? null : privObject.getTable().getDbName();
- objName = privObject.getTable() == null ? null : privObject.getTable().getTableName();
+ dbname = privObject.getTable().getDbName();
+ objName = privObject.getTable().getTableName();
+ columns = tableName2Cols == null ? null :
+ tableName2Cols.get(Table.getCompleteName(dbname, objName));
break;
case DFS_DIR:
case LOCAL_DIR:
@@ -788,7 +770,7 @@ public class Driver implements CommandPr
}
HivePrivObjectActionType actionType = AuthorizationUtils.getActionType(privObject);
HivePrivilegeObject hPrivObject = new HivePrivilegeObject(privObjType, dbname, objName,
- actionType);
+ partKeys, columns, actionType, null);
hivePrivobjs.add(hPrivObject);
}
return hivePrivobjs;
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Wed Aug 13 02:28:54 2014
@@ -35,7 +35,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -600,13 +599,9 @@ public class DDLTask extends Task<DDLWor
HiveAuthorizer authorizer = getSessionAuthorizer();
try {
- Set<String> colSet = showGrantDesc.getColumns() != null ? new HashSet<String>(
- showGrantDesc.getColumns()) : null;
List<HivePrivilegeInfo> privInfos = authorizer.showPrivileges(
AuthorizationUtils.getHivePrincipal(showGrantDesc.getPrincipalDesc()),
- AuthorizationUtils.getHivePrivilegeObject(showGrantDesc.getHiveObj(),
- colSet
- ));
+ AuthorizationUtils.getHivePrivilegeObject(showGrantDesc.getHiveObj()));
boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST);
writeToFile(writeGrantInfo(privInfos, testMode), showGrantDesc.getResFile());
} catch (IOException e) {
@@ -625,7 +620,7 @@ public class DDLTask extends Task<DDLWor
//Convert to object types used by the authorization plugin interface
List<HivePrincipal> hivePrincipals = AuthorizationUtils.getHivePrincipals(principals);
List<HivePrivilege> hivePrivileges = AuthorizationUtils.getHivePrivileges(privileges);
- HivePrivilegeObject hivePrivObject = AuthorizationUtils.getHivePrivilegeObject(privSubjectDesc, null);
+ HivePrivilegeObject hivePrivObject = AuthorizationUtils.getHivePrivilegeObject(privSubjectDesc);
HivePrincipal grantorPrincipal = new HivePrincipal(
grantor, AuthorizationUtils.getHivePrincipalType(grantorType));
@@ -754,8 +749,7 @@ public class DDLTask extends Task<DDLWor
}
private int dropIndex(Hive db, DropIndexDesc dropIdx) throws HiveException {
- db.dropIndex(SessionState.get().getCurrentDatabase(), dropIdx.getTableName(),
- dropIdx.getIndexName(), true);
+ db.dropIndex(dropIdx.getTableName(), dropIdx.getIndexName(), true);
return 0;
}
@@ -765,11 +759,7 @@ public class DDLTask extends Task<DDLWor
validateSerDe(crtIndex.getSerde());
}
- String indexTableName =
- crtIndex.getIndexTableName() != null ? crtIndex.getIndexTableName() :
- MetaStoreUtils.getIndexTableName(SessionState.get().getCurrentDatabase(),
- crtIndex.getTableName(), crtIndex.getIndexName());
-
+ String indexTableName = crtIndex.getIndexTableName();
if (!Utilities.isDefaultNameNode(conf)) {
// If location is specified - ensure that it is a full qualified name
makeLocationQualified(crtIndex, indexTableName);
@@ -792,10 +782,9 @@ public class DDLTask extends Task<DDLWor
}
private int alterIndex(Hive db, AlterIndexDesc alterIndex) throws HiveException {
- String dbName = alterIndex.getDbName();
String baseTableName = alterIndex.getBaseTableName();
String indexName = alterIndex.getIndexName();
- Index idx = db.getIndex(dbName, baseTableName, indexName);
+ Index idx = db.getIndex(baseTableName, indexName);
switch(alterIndex.getOp()) {
case ADDPROPS:
@@ -806,8 +795,7 @@ public class DDLTask extends Task<DDLWor
Map<String, String> props = new HashMap<String, String>();
Map<Map<String, String>, Long> basePartTs = new HashMap<Map<String, String>, Long>();
- Table baseTbl = db.getTable(SessionState.get().getCurrentDatabase(),
- baseTableName);
+ Table baseTbl = db.getTable(baseTableName);
if (baseTbl.isPartitioned()) {
List<Partition> baseParts;
@@ -854,7 +842,7 @@ public class DDLTask extends Task<DDLWor
}
try {
- db.alterIndex(dbName, baseTableName, indexName, idx);
+ db.alterIndex(baseTableName, indexName, idx);
} catch (InvalidOperationException e) {
console.printError("Invalid alter operation: " + e.getMessage());
LOG.info("alter index: " + stringifyException(e));
@@ -896,7 +884,7 @@ public class DDLTask extends Task<DDLWor
*/
private int renamePartition(Hive db, RenamePartitionDesc renamePartitionDesc) throws HiveException {
- Table tbl = db.getTable(renamePartitionDesc.getDbName(), renamePartitionDesc.getTableName());
+ Table tbl = db.getTable(renamePartitionDesc.getTableName());
Partition oldPart = db.getPartition(tbl, renamePartitionDesc.getOldPartSpec(), false);
Partition part = db.getPartition(tbl, renamePartitionDesc.getOldPartSpec(), false);
@@ -923,7 +911,7 @@ public class DDLTask extends Task<DDLWor
private int alterTableAlterPart(Hive db, AlterTableAlterPartDesc alterPartitionDesc)
throws HiveException {
- Table tbl = db.getTable(alterPartitionDesc.getDbName(), alterPartitionDesc.getTableName());
+ Table tbl = db.getTable(alterPartitionDesc.getTableName(), true);
String tabName = alterPartitionDesc.getTableName();
// This is checked by DDLSemanticAnalyzer
@@ -1015,14 +1003,11 @@ public class DDLTask extends Task<DDLWor
private int touch(Hive db, AlterTableSimpleDesc touchDesc)
throws HiveException {
- String dbName = touchDesc.getDbName();
- String tblName = touchDesc.getTableName();
-
- Table tbl = db.getTable(dbName, tblName);
+ Table tbl = db.getTable(touchDesc.getTableName());
if (touchDesc.getPartSpec() == null) {
try {
- db.alterTable(tblName, tbl);
+ db.alterTable(touchDesc.getTableName(), tbl);
} catch (InvalidOperationException e) {
throw new HiveException("Uable to update table");
}
@@ -1034,7 +1019,7 @@ public class DDLTask extends Task<DDLWor
throw new HiveException("Specified partition does not exist");
}
try {
- db.alterPartition(tblName, part);
+ db.alterPartition(touchDesc.getTableName(), part);
} catch (InvalidOperationException e) {
throw new HiveException(e);
}
@@ -1173,10 +1158,8 @@ public class DDLTask extends Task<DDLWor
private int archive(Hive db, AlterTableSimpleDesc simpleDesc,
DriverContext driverContext)
throws HiveException {
- String dbName = simpleDesc.getDbName();
- String tblName = simpleDesc.getTableName();
- Table tbl = db.getTable(dbName, tblName);
+ Table tbl = db.getTable(simpleDesc.getTableName());
if (tbl.getTableType() != TableType.MANAGED_TABLE) {
throw new HiveException("ARCHIVE can only be performed on managed tables");
@@ -1378,7 +1361,7 @@ public class DDLTask extends Task<DDLWor
authority.toString(),
harPartitionDir.getPath()); // make in Path to ensure no slash at the end
setArchived(p, harPath, partSpecInfo.values.size());
- db.alterPartition(tblName, p);
+ db.alterPartition(simpleDesc.getTableName(), p);
}
} catch (Exception e) {
throw new HiveException("Unable to change the partition info for HAR", e);
@@ -1399,10 +1382,8 @@ public class DDLTask extends Task<DDLWor
private int unarchive(Hive db, AlterTableSimpleDesc simpleDesc)
throws HiveException {
- String dbName = simpleDesc.getDbName();
- String tblName = simpleDesc.getTableName();
- Table tbl = db.getTable(dbName, tblName);
+ Table tbl = db.getTable(simpleDesc.getTableName());
// Means user specified a table, not a partition
if (simpleDesc.getPartSpec() == null) {
@@ -1587,7 +1568,7 @@ public class DDLTask extends Task<DDLWor
for(Partition p: partitions) {
setUnArchived(p);
try {
- db.alterPartition(tblName, p);
+ db.alterPartition(simpleDesc.getTableName(), p);
} catch (InvalidOperationException e) {
throw new HiveException(e);
}
@@ -1636,10 +1617,7 @@ public class DDLTask extends Task<DDLWor
private int compact(Hive db, AlterTableSimpleDesc desc) throws HiveException {
- String dbName = desc.getDbName();
- String tblName = desc.getTableName();
-
- Table tbl = db.getTable(dbName, tblName);
+ Table tbl = db.getTable(desc.getTableName());
String partName = null;
if (desc.getPartSpec() == null) {
@@ -1852,7 +1830,7 @@ public class DDLTask extends Task<DDLWor
final String ROW_FORMAT = "row_format";
final String TBL_LOCATION = "tbl_location";
final String TBL_PROPERTIES = "tbl_properties";
- boolean isHbaseTable = false;
+ boolean needsLocation = true;
StringBuilder createTab_str = new StringBuilder();
String tableName = showCreateTbl.getTableName();
@@ -1864,9 +1842,7 @@ public class DDLTask extends Task<DDLWor
FileSystem fs = resFile.getFileSystem(conf);
outStream = fs.create(resFile);
- if (tbl.getStorageHandler() != null) {
- isHbaseTable = tbl.getStorageHandler().toString().equals("org.apache.hadoop.hive.hbase.HBaseStorageHandler");
- }
+ needsLocation = doesTableNeedLocation(tbl);
if (tbl.isView()) {
String createTab_stmt = "CREATE VIEW `" + tableName + "` AS " + tbl.getViewExpandedText();
@@ -1883,7 +1859,7 @@ public class DDLTask extends Task<DDLWor
createTab_str.append("<" + LIST_PARTITIONS + ">\n");
createTab_str.append("<" + SORT_BUCKET + ">\n");
createTab_str.append("<" + ROW_FORMAT + ">\n");
- if (!isHbaseTable) {
+ if (needsLocation) {
createTab_str.append("LOCATION\n");
createTab_str.append("<" + TBL_LOCATION + ">\n");
}
@@ -2065,7 +2041,7 @@ public class DDLTask extends Task<DDLWor
createTab_stmt.add(SORT_BUCKET, tbl_sort_bucket);
createTab_stmt.add(ROW_FORMAT, tbl_row_format);
// Table location should not be printed with hbase backed tables
- if (!isHbaseTable) {
+ if (needsLocation) {
createTab_stmt.add(TBL_LOCATION, tbl_location);
}
createTab_stmt.add(TBL_PROPERTIES, tbl_properties);
@@ -2233,15 +2209,7 @@ public class DDLTask extends Task<DDLWor
public int showColumns(Hive db, ShowColumnsDesc showCols)
throws HiveException {
- String dbName = showCols.getDbName();
- String tableName = showCols.getTableName();
- Table table = null;
- if (dbName == null) {
- table = db.getTable(tableName);
- }
- else {
- table = db.getTable(dbName, tableName);
- }
+ Table table = db.getTable(showCols.getTableName());
// write the results in the file
DataOutputStream outStream = null;
@@ -3280,7 +3248,8 @@ public class DDLTask extends Task<DDLWor
Table oldTbl = tbl.copy();
if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.RENAME) {
- tbl.setTableName(alterTbl.getNewName());
+ tbl.setDbName(Utilities.getDatabaseName(alterTbl.getNewName()));
+ tbl.setTableName(Utilities.getTableName(alterTbl.getNewName()));
} else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDCOLS) {
List<FieldSchema> newCols = alterTbl.getNewCols();
List<FieldSchema> oldCols = tbl.getCols();
@@ -3933,7 +3902,7 @@ public class DDLTask extends Task<DDLWor
tbl.getTTable().getSd().setOutputFormat(tbl.getOutputFormatClass().getName());
}
- if (!Utilities.isDefaultNameNode(conf) && tbl.getTTable().getSd().isSetLocation()) {
+ if (!Utilities.isDefaultNameNode(conf) && doesTableNeedLocation(tbl)) {
// If location is specified - ensure that it is a full qualified name
makeLocationQualified(tbl.getDbName(), tbl.getTTable().getSd(), tbl.getTableName());
}
@@ -4309,10 +4278,12 @@ public class DDLTask extends Task<DDLWor
if (crtIndex.getLocation() == null) {
// Location is not set, leave it as-is if index doesn't belong to default DB
// Currently all indexes are created in current DB only
- if (db.getDatabaseCurrent().getName().equalsIgnoreCase(MetaStoreUtils.DEFAULT_DATABASE_NAME)) {
+ if (Utilities.getDatabaseName(name).equalsIgnoreCase(MetaStoreUtils.DEFAULT_DATABASE_NAME)) {
// Default database name path is always ignored, use METASTOREWAREHOUSE and object name
// instead
- path = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE), name.toLowerCase());
+ String warehouse = HiveConf.getVar(conf, ConfVars.METASTOREWAREHOUSE);
+ String tableName = Utilities.getTableName(name);
+ path = new Path(warehouse, tableName.toLowerCase());
}
}
else {
@@ -4341,4 +4312,15 @@ public class DDLTask extends Task<DDLWor
database.getName().toLowerCase() + ".db")));
}
}
+
+ private static boolean doesTableNeedLocation(Table tbl) {
+ // If we are ok with breaking compatibility of existing 3rd party StorageHandlers,
+ // this method could be moved to the HiveStorageHandler interface.
+ boolean retval = true;
+ if (tbl.getStorageHandler() != null) {
+ retval = !tbl.getStorageHandler().toString().equals(
+ "org.apache.hadoop.hive.hbase.HBaseStorageHandler");
+ }
+ return retval;
+ }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java Wed Aug 13 02:28:54 2014
@@ -181,7 +181,7 @@ public class HashTableSinkOperator exten
if (pos == posBigTableAlias) {
continue;
}
- mapJoinTables[pos] = new HashMapWrapper(hconf);
+ mapJoinTables[pos] = new HashMapWrapper(hconf, -1);
TableDesc valueTableDesc = conf.getValueTblFilteredDescs().get(pos);
SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), null);
SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null);
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Aug 13 02:28:54 2014
@@ -96,6 +96,7 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.hive.common.HiveInterruptCallback;
import org.apache.hadoop.hive.common.HiveInterruptUtils;
import org.apache.hadoop.hive.common.HiveStatsUtils;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.Warehouse;
@@ -1381,9 +1382,8 @@ public final class Utilities {
public static RCFile.Writer createRCFileWriter(JobConf jc, FileSystem fs, Path file,
boolean isCompressed, Progressable progressable) throws IOException {
CompressionCodec codec = null;
- Class<?> codecClass = null;
if (isCompressed) {
- codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
+ Class<?> codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
}
return new RCFile.Writer(fs, jc, file, progressable, codec);
@@ -1972,6 +1972,7 @@ public final class Utilities {
newPath.remove(oneurl);
}
}
+ JavaUtils.closeClassLoader(loader);
loader = new URLClassLoader(newPath.toArray(new URL[0]));
curThread.setContextClassLoader(loader);
@@ -2046,19 +2047,53 @@ public final class Utilities {
* @return String array with two elements, first is db name, second is table name
* @throws HiveException
*/
- public static String[] getDbTableName(String dbtable) throws HiveException{
- if(dbtable == null){
+ public static String[] getDbTableName(String dbtable) throws SemanticException {
+ return getDbTableName(SessionState.get().getCurrentDatabase(), dbtable);
+ }
+
+ public static String[] getDbTableName(String defaultDb, String dbtable) throws SemanticException {
+ if (dbtable == null) {
return new String[2];
}
String[] names = dbtable.split("\\.");
switch (names.length) {
- case 2:
- return names;
- case 1:
- return new String [] {SessionState.get().getCurrentDatabase(), dbtable};
- default:
- throw new HiveException(ErrorMsg.INVALID_TABLE_NAME, dbtable);
+ case 2:
+ return names;
+ case 1:
+ return new String [] {defaultDb, dbtable};
+ default:
+ throw new SemanticException(ErrorMsg.INVALID_TABLE_NAME, dbtable);
+ }
+ }
+
+ /**
+ * Accepts qualified name which is in the form of dbname.tablename and returns dbname from it
+ *
+ * @param dbTableName
+ * @return dbname
+ * @throws SemanticException input string is not qualified name
+ */
+ public static String getDatabaseName(String dbTableName) throws SemanticException {
+ String[] split = dbTableName.split("\\.");
+ if (split.length != 2) {
+ throw new SemanticException(ErrorMsg.INVALID_TABLE_NAME, dbTableName);
+ }
+ return split[0];
+ }
+
+ /**
+ * Accepts qualified name which is in the form of dbname.tablename and returns tablename from it
+ *
+ * @param dbTableName
+ * @return tablename
+ * @throws SemanticException input string is not qualified name
+ */
+ public static String getTableName(String dbTableName) throws SemanticException {
+ String[] split = dbTableName.split("\\.");
+ if (split.length != 2) {
+ throw new SemanticException(ErrorMsg.INVALID_TABLE_NAME, dbTableName);
}
+ return split[1];
}
public static void validateColumnNames(List<String> colNames, List<String> checkCols)
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java Wed Aug 13 02:28:54 2014
@@ -145,8 +145,7 @@ public final class BytesBytesMultiHashMa
private long[] refs;
private int startingHashBitCount, hashBitCount;
- private int metricPutConflict = 0, metricSameBitsDiffKey = 0,
- metricSameBitsSameKey = 0, metricDiffBits = 0;
+ private int metricPutConflict = 0, metricExpands = 0, metricExpandsUs = 0;
/** We have 39 bits to store list pointer from the first record; this is size limit */
final static long MAX_WB_SIZE = ((long)1) << 38;
@@ -430,16 +429,13 @@ public final class BytesBytesMultiHashMa
*/
private boolean isSameKey(byte[] key, int length, long ref, int hashCode) {
if (!compareHashBits(ref, hashCode)) {
- ++metricDiffBits;
return false; // Hash bits don't match.
}
writeBuffers.setReadPoint(getFirstRecordLengthsOffset(ref));
int valueLength = (int)writeBuffers.readVLong(), keyLength = (int)writeBuffers.readVLong();
long keyOffset = Ref.getOffset(ref) - (valueLength + keyLength);
// See the comment in the other isSameKey
- boolean result = writeBuffers.isEqual(key, length, keyOffset, keyLength);
- if (result) { ++metricSameBitsSameKey; } else { ++metricSameBitsDiffKey; }
- return result;
+ return writeBuffers.isEqual(key, length, keyOffset, keyLength);
}
private boolean compareHashBits(long ref, int hashCode) {
@@ -461,6 +457,7 @@ public final class BytesBytesMultiHashMa
}
private void expandAndRehash() {
+ long expandTime = System.nanoTime();
final long[] oldRefs = refs;
long capacity = refs.length << 1;
validateCapacity(capacity);
@@ -492,6 +489,9 @@ public final class BytesBytesMultiHashMa
this.largestNumberOfSteps = maxSteps;
this.hashBitCount = newHashBitCount;
this.resizeThreshold = (int)(capacity * loadFactor);
+ metricExpandsUs += (System.nanoTime() - expandTime);
+ ++metricExpands;
+
}
/**
@@ -703,11 +703,9 @@ public final class BytesBytesMultiHashMa
}
public void debugDumpMetrics() {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Map metrics: keys " + this.keysAssigned + ", write conflict " + metricPutConflict
- + ", write max dist " + largestNumberOfSteps + ", read neq " + metricDiffBits
- + ", read eq-eq " + metricSameBitsSameKey + ", read eq-neq " + metricSameBitsDiffKey);
- }
+ LOG.info("Map metrics: keys allocated " + this.refs.length +", keys assigned " + keysAssigned
+ + ", write conflict " + metricPutConflict + ", write max dist " + largestNumberOfSteps
+ + ", expanded " + metricExpands + " times in " + metricExpandsUs + "us");
}
private void debugDumpKeyProbe(long keyOffset, int keyLength, int hashCode, int finalSlot) {
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java Wed Aug 13 02:28:54 2014
@@ -71,25 +71,39 @@ public class HashMapWrapper extends Abst
}
public HashMapWrapper() {
- this(HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD.defaultIntVal,
- HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR.defaultFloatVal, false, false);
+ this(HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT.defaultFloatVal,
+ HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD.defaultIntVal,
+ HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR.defaultFloatVal, false, false, -1);
}
- public HashMapWrapper(Configuration hconf) {
- this(HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD),
+ public HashMapWrapper(Configuration hconf, long keyCount) {
+ this(HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT),
+ HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD),
HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR),
HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINLAZYHASHTABLE),
- HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDKEYS));
+ HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDKEYS), keyCount);
}
- private HashMapWrapper(
- int threshold, float loadFactor, boolean useLazyRows, boolean useOptimizedKeys) {
+ private HashMapWrapper(float keyCountAdj, int threshold, float loadFactor,
+ boolean useLazyRows, boolean useOptimizedKeys, long keyCount) {
super(createConstructorMetaData(threshold, loadFactor));
+ threshold = calculateTableSize(keyCountAdj, threshold, loadFactor, keyCount);
mHash = new HashMap<MapJoinKey, MapJoinRowContainer>(threshold, loadFactor);
this.useLazyRows = useLazyRows;
this.useOptimizedKeys = useOptimizedKeys;
}
+ public static int calculateTableSize(
+ float keyCountAdj, int threshold, float loadFactor, long keyCount) {
+ if (keyCount >= 0 && keyCountAdj != 0) {
+ // We have statistics for the table. Size appropriately.
+ threshold = (int)Math.ceil(keyCount / (keyCountAdj * loadFactor));
+ }
+ LOG.info("Key count from statistics is " + keyCount + "; setting map size to " + threshold);
+ return threshold;
+ }
+
+
@Override
public MapJoinRowContainer get(MapJoinKey key) {
return mHash.get(key);
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java Wed Aug 13 02:28:54 2014
@@ -59,15 +59,17 @@ public class MapJoinBytesTableContainer
private List<Object> EMPTY_LIST = new ArrayList<Object>(0);
- public MapJoinBytesTableContainer(Configuration hconf, MapJoinObjectSerDeContext valCtx)
- throws SerDeException {
- this(HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD),
+ public MapJoinBytesTableContainer(Configuration hconf,
+ MapJoinObjectSerDeContext valCtx, long keyCount) throws SerDeException {
+ this(HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT),
+ HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD),
HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR),
- HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE), valCtx);
+ HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE), valCtx, keyCount);
}
- private MapJoinBytesTableContainer(int threshold, float loadFactor, int wbSize,
- MapJoinObjectSerDeContext valCtx) throws SerDeException {
+ private MapJoinBytesTableContainer(float keyCountAdj, int threshold, float loadFactor,
+ int wbSize, MapJoinObjectSerDeContext valCtx, long keyCount) throws SerDeException {
+ threshold = HashMapWrapper.calculateTableSize(keyCountAdj, threshold, loadFactor, keyCount);
hashMap = new BytesBytesMultiHashMap(threshold, loadFactor, wbSize);
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java Wed Aug 13 02:28:54 2014
@@ -88,6 +88,7 @@ public class HashTableLoader implements
TezContext tezContext = (TezContext) MapredContext.get();
Map<Integer, String> parentToInput = desc.getParentToInput();
+ Map<Integer, Long> parentKeyCounts = desc.getParentKeyCounts();
boolean useOptimizedTables = HiveConf.getBoolVar(
hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE);
@@ -117,8 +118,11 @@ public class HashTableLoader implements
}
}
isFirstKey = false;
+ Long keyCountObj = parentKeyCounts.get(pos);
+ long keyCount = (keyCountObj == null) ? -1 : keyCountObj.longValue();
MapJoinTableContainer tableContainer = useOptimizedTables
- ? new MapJoinBytesTableContainer(hconf, valCtx) : new HashMapWrapper(hconf);
+ ? new MapJoinBytesTableContainer(hconf, valCtx, keyCount)
+ : new HashMapWrapper(hconf, keyCount);
while (kvReader.next()) {
lastKey = tableContainer.putRow(keyCtx, (Writable)kvReader.getCurrentKey(),
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Wed Aug 13 02:28:54 2014
@@ -80,6 +80,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.udf.SettableUDF;
import org.apache.hadoop.hive.ql.udf.UDFConv;
import org.apache.hadoop.hive.ql.udf.UDFHex;
@@ -323,10 +324,12 @@ public class VectorizationContext {
ve = getGenericUdfVectorExpression(expr.getGenericUDF(),
childExpressions, mode, exprDesc.getTypeInfo());
}
+ } else if (exprDesc instanceof ExprNodeNullDesc) {
+ ve = getConstantVectorExpression(null, exprDesc.getTypeInfo(), mode);
} else if (exprDesc instanceof ExprNodeConstantDesc) {
ve = getConstantVectorExpression(((ExprNodeConstantDesc) exprDesc).getValue(), exprDesc.getTypeInfo(),
mode);
- }
+ }
if (ve == null) {
throw new HiveException("Could not vectorize expression: "+exprDesc.getName());
}
@@ -410,8 +413,8 @@ public class VectorizationContext {
}
}
} else {
- for (ExprNodeDesc child : children) {
- ExprNodeDesc castExpression = getImplicitCastExpression(genericUDF, child, commonType);
+ for (ExprNodeDesc child : children) {
+ ExprNodeDesc castExpression = getImplicitCastExpression(genericUDF, child, commonType);
if (castExpression != null) {
atleastOneCastNeeded = true;
childrenWithCasts.add(castExpression);
@@ -652,69 +655,70 @@ public class VectorizationContext {
}
/**
- * Handles only the special case of unary operators on a constant.
+ * Handles only the special cases of cast/+ve/-ve operator on a constant.
* @param exprDesc
- * @return The same expression if no folding done, else return the constant
+ * @return The same expression if no evaluation done, else return the constant
* expression.
* @throws HiveException
*/
- ExprNodeDesc foldConstantsForUnaryExpression(ExprNodeDesc exprDesc) throws HiveException {
- if (!(exprDesc instanceof ExprNodeGenericFuncDesc)) {
- return exprDesc;
- }
-
- if (exprDesc.getChildren() == null || (exprDesc.getChildren().size() != 1) ) {
- return exprDesc;
- }
-
- ExprNodeConstantDesc foldedChild = null;
- if (!( exprDesc.getChildren().get(0) instanceof ExprNodeConstantDesc)) {
-
- // try recursive folding
- ExprNodeDesc expr = foldConstantsForUnaryExpression(exprDesc.getChildren().get(0));
- if (expr instanceof ExprNodeConstantDesc) {
- foldedChild = (ExprNodeConstantDesc) expr;
- }
- } else {
- foldedChild = (ExprNodeConstantDesc) exprDesc.getChildren().get(0);
- }
-
- if (foldedChild == null) {
- return exprDesc;
- }
-
- ObjectInspector childoi = foldedChild.getWritableObjectInspector();
- GenericUDF gudf = ((ExprNodeGenericFuncDesc) exprDesc).getGenericUDF();
-
- if (gudf instanceof GenericUDFOPNegative || gudf instanceof GenericUDFOPPositive
- || castExpressionUdfs.contains(gudf.getClass())
- || ((gudf instanceof GenericUDFBridge)
- && castExpressionUdfs.contains(((GenericUDFBridge) gudf).getUdfClass()))) {
- ExprNodeEvaluator<?> evaluator = ExprNodeEvaluatorFactory.get(exprDesc);
- ObjectInspector output = evaluator.initialize(childoi);
- Object constant = evaluator.evaluate(null);
- Object java = ObjectInspectorUtils.copyToStandardJavaObject(constant, output);
- return new ExprNodeConstantDesc(exprDesc.getTypeInfo(), java);
- }
-
- return exprDesc;
+ ExprNodeDesc evaluateCastOnConstants(ExprNodeDesc exprDesc) throws HiveException {
+ if (!(exprDesc instanceof ExprNodeGenericFuncDesc)) {
+ return exprDesc;
+ }
+
+ if (exprDesc.getChildren() == null || (exprDesc.getChildren().size() != 1) ) {
+ return exprDesc;
+ }
+
+ ExprNodeConstantDesc foldedChild = null;
+ if (!( exprDesc.getChildren().get(0) instanceof ExprNodeConstantDesc)) {
+
+ // try recursive folding
+ ExprNodeDesc expr = evaluateCastOnConstants(exprDesc.getChildren().get(0));
+ if (expr instanceof ExprNodeConstantDesc) {
+ foldedChild = (ExprNodeConstantDesc) expr;
+ }
+ } else {
+ foldedChild = (ExprNodeConstantDesc) exprDesc.getChildren().get(0);
+ }
+
+ if (foldedChild == null) {
+ return exprDesc;
+ }
+
+ ObjectInspector childoi = foldedChild.getWritableObjectInspector();
+ GenericUDF gudf = ((ExprNodeGenericFuncDesc) exprDesc).getGenericUDF();
+
+ // Only evaluate +ve/-ve or cast on constant or recursive casting.
+ if (gudf instanceof GenericUDFOPNegative || gudf instanceof GenericUDFOPPositive ||
+ castExpressionUdfs.contains(gudf.getClass())
+ || ((gudf instanceof GenericUDFBridge)
+ && castExpressionUdfs.contains(((GenericUDFBridge) gudf).getUdfClass()))) {
+ ExprNodeEvaluator<?> evaluator = ExprNodeEvaluatorFactory.get(exprDesc);
+ ObjectInspector output = evaluator.initialize(childoi);
+ Object constant = evaluator.evaluate(null);
+ Object java = ObjectInspectorUtils.copyToStandardJavaObject(constant, output);
+ return new ExprNodeConstantDesc(exprDesc.getTypeInfo(), java);
+ }
+
+ return exprDesc;
}
-
- /* Fold simple unary expressions in all members of the input list and return new list
+
+ /* For cast on constant operator in all members of the input list and return new list
* containing results.
*/
- private List<ExprNodeDesc> foldConstantsForUnaryExprs(List<ExprNodeDesc> childExpr)
- throws HiveException {
- List<ExprNodeDesc> constantFoldedChildren = new ArrayList<ExprNodeDesc>();
- if (childExpr != null) {
- for (ExprNodeDesc expr : childExpr) {
- expr = this.foldConstantsForUnaryExpression(expr);
- constantFoldedChildren.add(expr);
- }
- }
- return constantFoldedChildren;
+ private List<ExprNodeDesc> evaluateCastOnConstants(List<ExprNodeDesc> childExpr)
+ throws HiveException {
+ List<ExprNodeDesc> evaluatedChildren = new ArrayList<ExprNodeDesc>();
+ if (childExpr != null) {
+ for (ExprNodeDesc expr : childExpr) {
+ expr = this.evaluateCastOnConstants(expr);
+ evaluatedChildren.add(expr);
+ }
+ }
+ return evaluatedChildren;
}
-
+
private VectorExpression getConstantVectorExpression(Object constantValue, TypeInfo typeInfo,
Mode mode) throws HiveException {
String type = typeInfo.getTypeName();
@@ -903,8 +907,9 @@ public class VectorizationContext {
private VectorExpression getGenericUdfVectorExpression(GenericUDF udf,
List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType) throws HiveException {
- List<ExprNodeDesc> constantFoldedChildren = foldConstantsForUnaryExprs(childExpr);
- childExpr = constantFoldedChildren;
+ List<ExprNodeDesc> castedChildren = evaluateCastOnConstants(childExpr);
+ childExpr = castedChildren;
+
//First handle special cases
if (udf instanceof GenericUDFBetween) {
return getBetweenFilterExpression(childExpr, mode, returnType);
@@ -928,15 +933,15 @@ public class VectorizationContext {
}
} else if (udf instanceof GenericUDFToDecimal) {
return getCastToDecimal(childExpr, returnType);
- }
-
+ }
+
// Now do a general lookup
Class<?> udfClass = udf.getClass();
if (udf instanceof GenericUDFBridge) {
udfClass = ((GenericUDFBridge) udf).getUdfClass();
}
- VectorExpression ve = getVectorExpressionForUdf(udfClass, constantFoldedChildren, mode, returnType);
+ VectorExpression ve = getVectorExpressionForUdf(udfClass, castedChildren, mode, returnType);
if (ve == null) {
throw new HiveException("Udf: "+udf.getClass().getSimpleName()+", is not supported");
@@ -998,7 +1003,7 @@ public class VectorizationContext {
}
}
}
-
+
/**
* Create a filter or boolean-valued expression for column IN ( <list-of-constants> )
*/
@@ -1006,13 +1011,11 @@ public class VectorizationContext {
throws HiveException {
ExprNodeDesc colExpr = childExpr.get(0);
- TypeInfo colTypeInfo = colExpr.getTypeInfo();
String colType = colExpr.getTypeString();
// prepare arguments for createVectorExpression
- List<ExprNodeDesc> childrenForInList =
- foldConstantsForUnaryExprs(childExpr.subList(1, childExpr.size()));
-
+ List<ExprNodeDesc> childrenForInList = evaluateCastOnConstants(childExpr.subList(1, childExpr.size()));
+
/* This method assumes that the IN list has no NULL entries. That is enforced elsewhere,
* in the Vectorizer class. If NULL is passed in as a list entry, behavior is not defined.
* If in the future, NULL values are allowed in the IN list, be sure to handle 3-valued
@@ -1107,16 +1110,116 @@ public class VectorizationContext {
return getCastToString(childExpr, returnType);
}
return null;
- }
-
+ }
+
+ private Decimal128 castConstantToDecimal(Object scalar, TypeInfo type) throws HiveException {
+ PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
+ String typename = type.getTypeName();
+ Decimal128 d = new Decimal128();
+ int scale = HiveDecimalUtils.getScaleForType(ptinfo);
+ switch (ptinfo.getPrimitiveCategory()) {
+ case FLOAT:
+ float floatVal = ((Float) scalar).floatValue();
+ d.update(floatVal, (short) scale);
+ break;
+ case DOUBLE:
+ double doubleVal = ((Double) scalar).doubleValue();
+ d.update(doubleVal, (short) scale);
+ break;
+ case BYTE:
+ byte byteVal = ((Byte) scalar).byteValue();
+ d.update(byteVal, (short) scale);
+ break;
+ case SHORT:
+ short shortVal = ((Short) scalar).shortValue();
+ d.update(shortVal, (short) scale);
+ break;
+ case INT:
+ int intVal = ((Integer) scalar).intValue();
+ d.update(intVal, (short) scale);
+ break;
+ case LONG:
+ long longVal = ((Long) scalar).longValue();
+ d.update(longVal, (short) scale);
+ break;
+ case DECIMAL:
+ HiveDecimal decimalVal = (HiveDecimal) scalar;
+ d.update(decimalVal.unscaledValue(), (short) scale);
+ break;
+ default:
+ throw new HiveException("Unsupported type "+typename+" for cast to Decimal128");
+ }
+ return d;
+ }
+
+ private String castConstantToString(Object scalar, TypeInfo type) throws HiveException {
+ PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
+ String typename = type.getTypeName();
+ switch (ptinfo.getPrimitiveCategory()) {
+ case FLOAT:
+ case DOUBLE:
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ return ((Number) scalar).toString();
+ case DECIMAL:
+ HiveDecimal decimalVal = (HiveDecimal) scalar;
+ return decimalVal.toString();
+ default:
+ throw new HiveException("Unsupported type "+typename+" for cast to String");
+ }
+ }
+
+ private Double castConstantToDouble(Object scalar, TypeInfo type) throws HiveException {
+ PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
+ String typename = type.getTypeName();
+ switch (ptinfo.getPrimitiveCategory()) {
+ case FLOAT:
+ case DOUBLE:
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ return ((Number) scalar).doubleValue();
+ case DECIMAL:
+ HiveDecimal decimalVal = (HiveDecimal) scalar;
+ return decimalVal.doubleValue();
+ default:
+ throw new HiveException("Unsupported type "+typename+" for cast to Double");
+ }
+ }
+
+ private Long castConstantToLong(Object scalar, TypeInfo type) throws HiveException {
+ PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
+ String typename = type.getTypeName();
+ switch (ptinfo.getPrimitiveCategory()) {
+ case FLOAT:
+ case DOUBLE:
+ case BYTE:
+ case SHORT:
+ case INT:
+ case LONG:
+ return ((Number) scalar).longValue();
+ case DECIMAL:
+ HiveDecimal decimalVal = (HiveDecimal) scalar;
+ return decimalVal.longValue();
+ default:
+ throw new HiveException("Unsupported type "+typename+" for cast to Long");
+ }
+ }
+
private VectorExpression getCastToDecimal(List<ExprNodeDesc> childExpr, TypeInfo returnType)
throws HiveException {
ExprNodeDesc child = childExpr.get(0);
String inputType = childExpr.get(0).getTypeString();
if (child instanceof ExprNodeConstantDesc) {
- // Don't do constant folding here. Wait until the optimizer is changed to do it.
- // Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
- return null;
+ // Return a constant vector expression
+ Object constantValue = ((ExprNodeConstantDesc) child).getValue();
+ Decimal128 decimalValue = castConstantToDecimal(constantValue, child.getTypeInfo());
+ return getConstantVectorExpression(decimalValue, returnType, Mode.PROJECTION);
+ } else if (child instanceof ExprNodeNullDesc) {
+ return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
}
if (isIntFamily(inputType)) {
return createVectorExpression(CastLongToDecimal.class, childExpr, Mode.PROJECTION, returnType);
@@ -1131,16 +1234,19 @@ public class VectorizationContext {
return createVectorExpression(CastTimestampToDecimal.class, childExpr, Mode.PROJECTION, returnType);
}
throw new HiveException("Unhandled cast input type: " + inputType);
- }
-
+ }
+
private VectorExpression getCastToString(List<ExprNodeDesc> childExpr, TypeInfo returnType)
throws HiveException {
ExprNodeDesc child = childExpr.get(0);
String inputType = childExpr.get(0).getTypeString();
if (child instanceof ExprNodeConstantDesc) {
- // Don't do constant folding here. Wait until the optimizer is changed to do it.
- // Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
- return null;
+ // Return a constant vector expression
+ Object constantValue = ((ExprNodeConstantDesc) child).getValue();
+ String strValue = castConstantToString(constantValue, child.getTypeInfo());
+ return getConstantVectorExpression(strValue, returnType, Mode.PROJECTION);
+ } else if (child instanceof ExprNodeNullDesc) {
+ return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
}
if (inputType.equals("boolean")) {
// Boolean must come before the integer family. It's a special case.
@@ -1164,9 +1270,12 @@ public class VectorizationContext {
ExprNodeDesc child = childExpr.get(0);
String inputType = childExpr.get(0).getTypeString();
if (child instanceof ExprNodeConstantDesc) {
- // Don't do constant folding here. Wait until the optimizer is changed to do it.
- // Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
- return null;
+ // Return a constant vector expression
+ Object constantValue = ((ExprNodeConstantDesc) child).getValue();
+ Double doubleValue = castConstantToDouble(constantValue, child.getTypeInfo());
+ return getConstantVectorExpression(doubleValue, returnType, Mode.PROJECTION);
+ } else if (child instanceof ExprNodeNullDesc) {
+ return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
}
if (isIntFamily(inputType)) {
return createVectorExpression(CastLongToDouble.class, childExpr, Mode.PROJECTION, returnType);
@@ -1191,6 +1300,8 @@ public class VectorizationContext {
// Don't do constant folding here. Wait until the optimizer is changed to do it.
// Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
return null;
+ } else if (child instanceof ExprNodeNullDesc) {
+ return getConstantVectorExpression(null, TypeInfoFactory.booleanTypeInfo, Mode.PROJECTION);
}
// Long and double are handled using descriptors, string needs to be specially handled.
if (inputType.equals("string")) {
@@ -1215,9 +1326,12 @@ public class VectorizationContext {
ExprNodeDesc child = childExpr.get(0);
String inputType = childExpr.get(0).getTypeString();
if (child instanceof ExprNodeConstantDesc) {
- // Don't do constant folding here. Wait until the optimizer is changed to do it.
- // Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
- return null;
+ // Return a constant vector expression
+ Object constantValue = ((ExprNodeConstantDesc) child).getValue();
+ Long longValue = castConstantToLong(constantValue, child.getTypeInfo());
+ return getConstantVectorExpression(longValue, TypeInfoFactory.longTypeInfo, Mode.PROJECTION);
+ } else if (child instanceof ExprNodeNullDesc) {
+ return getConstantVectorExpression(null, TypeInfoFactory.longTypeInfo, Mode.PROJECTION);
}
// Float family, timestamp are handled via descriptor based lookup, int family needs
// special handling.
@@ -1281,7 +1395,7 @@ public class VectorizationContext {
String colType = commonType.getTypeName();
// prepare arguments for createVectorExpression
- List<ExprNodeDesc> childrenAfterNot = foldConstantsForUnaryExprs(castChildren);
+ List<ExprNodeDesc> childrenAfterNot = evaluateCastOnConstants(castChildren);
// determine class
Class<?> cl = null;
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java Wed Aug 13 02:28:54 2014
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
@@ -123,6 +124,7 @@ public class VectorizedBatchUtil {
case DOUBLE:
cvList.add(new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE));
break;
+ case BINARY:
case STRING:
cvList.add(new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE));
break;
@@ -237,7 +239,7 @@ public class VectorizedBatchUtil {
// float/double. String types have no default value for null.
switch (poi.getPrimitiveCategory()) {
case BOOLEAN: {
- LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+ LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
if (writableCol != null) {
lcv.vector[rowIndex] = ((BooleanWritable) writableCol).get() ? 1 : 0;
lcv.isNull[rowIndex] = false;
@@ -248,7 +250,7 @@ public class VectorizedBatchUtil {
}
break;
case BYTE: {
- LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+ LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
if (writableCol != null) {
lcv.vector[rowIndex] = ((ByteWritable) writableCol).get();
lcv.isNull[rowIndex] = false;
@@ -259,7 +261,7 @@ public class VectorizedBatchUtil {
}
break;
case SHORT: {
- LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+ LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
if (writableCol != null) {
lcv.vector[rowIndex] = ((ShortWritable) writableCol).get();
lcv.isNull[rowIndex] = false;
@@ -270,7 +272,7 @@ public class VectorizedBatchUtil {
}
break;
case INT: {
- LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+ LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
if (writableCol != null) {
lcv.vector[rowIndex] = ((IntWritable) writableCol).get();
lcv.isNull[rowIndex] = false;
@@ -281,7 +283,7 @@ public class VectorizedBatchUtil {
}
break;
case LONG: {
- LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+ LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
if (writableCol != null) {
lcv.vector[rowIndex] = ((LongWritable) writableCol).get();
lcv.isNull[rowIndex] = false;
@@ -292,7 +294,7 @@ public class VectorizedBatchUtil {
}
break;
case DATE: {
- LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+ LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
if (writableCol != null) {
lcv.vector[rowIndex] = ((DateWritable) writableCol).getDays();
lcv.isNull[rowIndex] = false;
@@ -303,7 +305,7 @@ public class VectorizedBatchUtil {
}
break;
case FLOAT: {
- DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[off+i];
+ DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[off + i];
if (writableCol != null) {
dcv.vector[rowIndex] = ((FloatWritable) writableCol).get();
dcv.isNull[rowIndex] = false;
@@ -314,7 +316,7 @@ public class VectorizedBatchUtil {
}
break;
case DOUBLE: {
- DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[off+i];
+ DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[off + i];
if (writableCol != null) {
dcv.vector[rowIndex] = ((DoubleWritable) writableCol).get();
dcv.isNull[rowIndex] = false;
@@ -325,7 +327,7 @@ public class VectorizedBatchUtil {
}
break;
case TIMESTAMP: {
- LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+ LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
if (writableCol != null) {
Timestamp t = ((TimestampWritable) writableCol).getTimestamp();
lcv.vector[rowIndex] = TimestampUtils.getTimeNanoSec(t);
@@ -336,8 +338,27 @@ public class VectorizedBatchUtil {
}
}
break;
+ case BINARY: {
+ BytesColumnVector bcv = (BytesColumnVector) batch.cols[off + i];
+ if (writableCol != null) {
+ bcv.isNull[rowIndex] = false;
+ BytesWritable bw = (BytesWritable) writableCol;
+ byte[] bytes = bw.getBytes();
+ int start = buffer.getLength();
+ int length = bytes.length;
+ try {
+ buffer.write(bytes, 0, length);
+ } catch (IOException ioe) {
+ throw new IllegalStateException("bad write", ioe);
+ }
+ bcv.setRef(rowIndex, buffer.getData(), start, length);
+ } else {
+ setNullColIsNullValue(bcv, rowIndex);
+ }
+ }
+ break;
case STRING: {
- BytesColumnVector bcv = (BytesColumnVector) batch.cols[off+i];
+ BytesColumnVector bcv = (BytesColumnVector) batch.cols[off + i];
if (writableCol != null) {
bcv.isNull[rowIndex] = false;
Text colText = (Text) writableCol;
@@ -355,7 +376,7 @@ public class VectorizedBatchUtil {
}
break;
case DECIMAL:
- DecimalColumnVector dcv = (DecimalColumnVector) batch.cols[off+i];
+ DecimalColumnVector dcv = (DecimalColumnVector) batch.cols[off + i];
if (writableCol != null) {
dcv.isNull[rowIndex] = false;
HiveDecimalWritable wobj = (HiveDecimalWritable) writableCol;
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java Wed Aug 13 02:28:54 2014
@@ -152,13 +152,20 @@ public class VectorizedColumnarSerDe ext
ByteBuffer b = Text.encode(String.valueOf(dcv.vector[rowIndex]));
serializeVectorStream.write(b.array(), 0, b.limit());
break;
- case STRING:
+ case BINARY: {
+ BytesColumnVector bcv = (BytesColumnVector) batch.cols[k];
+ byte[] bytes = bcv.vector[rowIndex];
+ serializeVectorStream.write(bytes, 0, bytes.length);
+ }
+ break;
+ case STRING: {
BytesColumnVector bcv = (BytesColumnVector) batch.cols[k];
LazyUtils.writeEscaped(serializeVectorStream, bcv.vector[rowIndex],
bcv.start[rowIndex],
bcv.length[rowIndex],
serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams
.getNeedsEscape());
+ }
break;
case TIMESTAMP:
LongColumnVector tcv = (LongColumnVector) batch.cols[k];
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java Wed Aug 13 02:28:54 2014
@@ -278,7 +278,7 @@ public class VectorizedRowBatchCtx {
case PRIMITIVE: {
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) foi;
// Vectorization currently only supports the following data types:
- // BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP,
+ // BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, BINARY, STRING, TIMESTAMP,
// DATE and DECIMAL
switch (poi.getPrimitiveCategory()) {
case BOOLEAN:
@@ -294,6 +294,7 @@ public class VectorizedRowBatchCtx {
case DOUBLE:
result.cols[j] = new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
break;
+ case BINARY:
case STRING:
result.cols[j] = new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
break;
@@ -404,7 +405,7 @@ public class VectorizedRowBatchCtx {
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
- lcv.fill((Boolean)value == true ? 1 : 0);
+ lcv.fill((Boolean) value == true ? 1 : 0);
lcv.isNull[0] = false;
}
}
@@ -417,7 +418,7 @@ public class VectorizedRowBatchCtx {
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
- lcv.fill((Byte)value);
+ lcv.fill((Byte) value);
lcv.isNull[0] = false;
}
}
@@ -430,7 +431,7 @@ public class VectorizedRowBatchCtx {
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
- lcv.fill((Short)value);
+ lcv.fill((Short) value);
lcv.isNull[0] = false;
}
}
@@ -443,7 +444,7 @@ public class VectorizedRowBatchCtx {
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
- lcv.fill((Integer)value);
+ lcv.fill((Integer) value);
lcv.isNull[0] = false;
}
}
@@ -456,7 +457,7 @@ public class VectorizedRowBatchCtx {
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
- lcv.fill((Long)value);
+ lcv.fill((Long) value);
lcv.isNull[0] = false;
}
}
@@ -469,7 +470,7 @@ public class VectorizedRowBatchCtx {
lcv.isNull[0] = true;
lcv.isRepeating = true;
} else {
- lcv.fill(((Date)value).getTime());
+ lcv.fill(((Date) value).getTime());
lcv.isNull[0] = false;
}
}
@@ -521,17 +522,31 @@ public class VectorizedRowBatchCtx {
dv.isNull[0] = true;
dv.isRepeating = true;
} else {
- HiveDecimal hd = (HiveDecimal)(value);
- dv.vector[0] = new Decimal128(hd.toString(), (short)hd.scale());
+ HiveDecimal hd = (HiveDecimal) value;
+ dv.vector[0] = new Decimal128(hd.toString(), (short) hd.scale());
dv.isRepeating = true;
dv.isNull[0] = false;
}
}
break;
-
+
+ case BINARY: {
+ BytesColumnVector bcv = (BytesColumnVector) batch.cols[colIndex];
+ byte[] bytes = (byte[]) value;
+ if (bytes == null) {
+ bcv.noNulls = false;
+ bcv.isNull[0] = true;
+ bcv.isRepeating = true;
+ } else {
+ bcv.fill(bytes);
+ bcv.isNull[0] = false;
+ }
+ }
+ break;
+
case STRING: {
BytesColumnVector bcv = (BytesColumnVector) batch.cols[colIndex];
- String sVal = (String)value;
+ String sVal = (String) value;
if (sVal == null) {
bcv.noNulls = false;
bcv.isNull[0] = true;
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java Wed Aug 13 02:28:54 2014
@@ -187,13 +187,14 @@ public class ConstantVectorExpression ex
public void setTypeString(String typeString) {
this.outputType = typeString;
- if ("string".equalsIgnoreCase(typeString)) {
+ if (VectorizationContext.isStringFamily(typeString)) {
this.type = Type.BYTES;
- } else if ("double".equalsIgnoreCase(typeString)) {
+ } else if (VectorizationContext.isFloatFamily(typeString)) {
this.type = Type.DOUBLE;
- } else if (VectorizationContext.decimalTypePattern.matcher(typeString).matches()){
+ } else if (VectorizationContext.isDecimalFamily(typeString)){
this.type = Type.DECIMAL;
} else {
+ // everything else that does not belong to string, double, decimal is treated as long.
this.type = Type.LONG;
}
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java Wed Aug 13 02:28:54 2014
@@ -50,25 +50,12 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableStringObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
/**
* VectorExpressionWritableFactory helper class for generating VectorExpressionWritable objects.
@@ -364,7 +351,6 @@ public final class VectorExpressionWrite
*/
public static VectorExpressionWriter genVectorExpressionWritable(ExprNodeDesc nodeDesc)
throws HiveException {
- String nodeType = nodeDesc.getTypeString();
ObjectInspector objectInspector = nodeDesc.getWritableObjectInspector();
if (null == objectInspector) {
objectInspector = TypeInfoUtils
@@ -408,6 +394,9 @@ public final class VectorExpressionWrite
case LONG:
return genVectorExpressionWritableLong(
(SettableLongObjectInspector) fieldObjInspector);
+ case VOID:
+ return genVectorExpressionWritableVoid(
+ (VoidObjectInspector) fieldObjInspector);
case BINARY:
return genVectorExpressionWritableBinary(
(SettableBinaryObjectInspector) fieldObjInspector);
@@ -722,6 +711,39 @@ public final class VectorExpressionWrite
}.init(fieldObjInspector);
}
+ private static VectorExpressionWriter genVectorExpressionWritableVoid(
+ VoidObjectInspector fieldObjInspector) throws HiveException {
+ return new VectorExpressionWriterLong() {
+ private Object obj;
+
+ public VectorExpressionWriter init(VoidObjectInspector objInspector)
+ throws HiveException {
+ super.init(objInspector);
+ this.obj = initValue(null);
+ return this;
+ }
+
+ @Override
+ public Object writeValue(long value) throws HiveException {
+ return this.obj;
+ }
+
+ @Override
+ public Object setValue(Object field, long value) throws HiveException {
+ if (null == field) {
+ field = initValue(null);
+ }
+ return field;
+ }
+
+ @Override
+ public Object initValue(Object ignored) {
+ return ((VoidObjectInspector) this.objectInspector).copyObject(null);
+ }
+ }.init(fieldObjInspector);
+ }
+
+
private static VectorExpressionWriter genVectorExpressionWritableInt(
SettableIntObjectInspector fieldObjInspector) throws HiveException {
return new VectorExpressionWriterLong() {