You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/07/09 11:07:43 UTC
svn commit: r1501145 [3/13] - in /hive/branches/vectorization: ./
beeline/src/java/org/apache/hive/beeline/
beeline/src/test/org/apache/hive/beeline/src/test/
cli/src/java/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/common/metric...
Modified: hive/branches/vectorization/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java (original)
+++ hive/branches/vectorization/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java Tue Jul 9 09:07:35 2013
@@ -47102,17 +47102,17 @@ public class ThriftHiveMetastore {
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.PARTITION_SPECS, new org.apache.thrift.meta_data.FieldMetaData("partitionSpecs", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
+ tmpMap.put(_Fields.PARTITION_SPECS, new org.apache.thrift.meta_data.FieldMetaData("partitionSpecs", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.SOURCE_DB, new org.apache.thrift.meta_data.FieldMetaData("source_db", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ tmpMap.put(_Fields.SOURCE_DB, new org.apache.thrift.meta_data.FieldMetaData("source_db", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.SOURCE_TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("source_table_name", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ tmpMap.put(_Fields.SOURCE_TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("source_table_name", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.DEST_DB, new org.apache.thrift.meta_data.FieldMetaData("dest_db", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ tmpMap.put(_Fields.DEST_DB, new org.apache.thrift.meta_data.FieldMetaData("dest_db", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.DEST_TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("dest_table_name", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ tmpMap.put(_Fields.DEST_TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("dest_table_name", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(exchange_partition_args.class, metaDataMap);
@@ -47945,9 +47945,9 @@ public class ThriftHiveMetastore {
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
tmpMap.put(_Fields.O2, new org.apache.thrift.meta_data.FieldMetaData("o2", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.O3, new org.apache.thrift.meta_data.FieldMetaData("o3", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ tmpMap.put(_Fields.O3, new org.apache.thrift.meta_data.FieldMetaData("o3", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.O4, new org.apache.thrift.meta_data.FieldMetaData("o4", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ tmpMap.put(_Fields.O4, new org.apache.thrift.meta_data.FieldMetaData("o4", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(exchange_partition_result.class, metaDataMap);
@@ -48459,7 +48459,7 @@ public class ThriftHiveMetastore {
while (true)
{
schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
break;
}
switch (schemeField.id) {
@@ -48468,7 +48468,7 @@ public class ThriftHiveMetastore {
struct.success = new Partition();
struct.success.read(iprot);
struct.setSuccessIsSet(true);
- } else {
+ } else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
@@ -48477,7 +48477,7 @@ public class ThriftHiveMetastore {
struct.o1 = new MetaException();
struct.o1.read(iprot);
struct.setO1IsSet(true);
- } else {
+ } else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
@@ -48486,7 +48486,7 @@ public class ThriftHiveMetastore {
struct.o2 = new NoSuchObjectException();
struct.o2.read(iprot);
struct.setO2IsSet(true);
- } else {
+ } else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
@@ -48495,7 +48495,7 @@ public class ThriftHiveMetastore {
struct.o3 = new InvalidObjectException();
struct.o3.read(iprot);
struct.setO3IsSet(true);
- } else {
+ } else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
@@ -48504,7 +48504,7 @@ public class ThriftHiveMetastore {
struct.o4 = new InvalidInputException();
struct.o4.read(iprot);
struct.setO4IsSet(true);
- } else {
+ } else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
@@ -48726,17 +48726,17 @@ public class ThriftHiveMetastore {
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.DB_NAME, new org.apache.thrift.meta_data.FieldMetaData("db_name", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ tmpMap.put(_Fields.DB_NAME, new org.apache.thrift.meta_data.FieldMetaData("db_name", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.TBL_NAME, new org.apache.thrift.meta_data.FieldMetaData("tbl_name", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ tmpMap.put(_Fields.TBL_NAME, new org.apache.thrift.meta_data.FieldMetaData("tbl_name", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.PART_VALS, new org.apache.thrift.meta_data.FieldMetaData("part_vals", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
+ tmpMap.put(_Fields.PART_VALS, new org.apache.thrift.meta_data.FieldMetaData("part_vals", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
- tmpMap.put(_Fields.USER_NAME, new org.apache.thrift.meta_data.FieldMetaData("user_name", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ tmpMap.put(_Fields.USER_NAME, new org.apache.thrift.meta_data.FieldMetaData("user_name", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.GROUP_NAMES, new org.apache.thrift.meta_data.FieldMetaData("group_names", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
+ tmpMap.put(_Fields.GROUP_NAMES, new org.apache.thrift.meta_data.FieldMetaData("group_names", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_partition_with_auth_args.class, metaDataMap);
@@ -49283,7 +49283,7 @@ public class ThriftHiveMetastore {
while (true)
{
schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
break;
}
switch (schemeField.id) {
@@ -49291,7 +49291,7 @@ public class ThriftHiveMetastore {
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.db_name = iprot.readString();
struct.setDb_nameIsSet(true);
- } else {
+ } else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
@@ -49299,7 +49299,7 @@ public class ThriftHiveMetastore {
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.tbl_name = iprot.readString();
struct.setTbl_nameIsSet(true);
- } else {
+ } else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
@@ -49317,7 +49317,7 @@ public class ThriftHiveMetastore {
iprot.readListEnd();
}
struct.setPart_valsIsSet(true);
- } else {
+ } else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
@@ -49325,7 +49325,7 @@ public class ThriftHiveMetastore {
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.user_name = iprot.readString();
struct.setUser_nameIsSet(true);
- } else {
+ } else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
@@ -49343,7 +49343,7 @@ public class ThriftHiveMetastore {
iprot.readListEnd();
}
struct.setGroup_namesIsSet(true);
- } else {
+ } else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
@@ -49594,11 +49594,11 @@ public class ThriftHiveMetastore {
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, Partition.class)));
- tmpMap.put(_Fields.O1, new org.apache.thrift.meta_data.FieldMetaData("o1", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ tmpMap.put(_Fields.O1, new org.apache.thrift.meta_data.FieldMetaData("o1", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.O2, new org.apache.thrift.meta_data.FieldMetaData("o2", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ tmpMap.put(_Fields.O2, new org.apache.thrift.meta_data.FieldMetaData("o2", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_partition_with_auth_result.class, metaDataMap);
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Tue Jul 9 09:07:35 2013
@@ -258,7 +258,7 @@ public class ObjectStore implements RawS
return prop;
}
- private static PersistenceManagerFactory getPMF() {
+ private static synchronized PersistenceManagerFactory getPMF() {
if (pmf == null) {
pmf = JDOHelper.getPersistenceManagerFactory(prop);
DataStoreCache dsc = pmf.getDataStoreCache();
Modified: hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java (original)
+++ hive/branches/vectorization/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java Tue Jul 9 09:07:35 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.common.cla
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -126,6 +127,12 @@ public class RetryingHMSHandler implemen
// Due to reflection, the jdo exception is wrapped in
// invocationTargetException
caughtException = e.getCause();
+ } else if (e.getCause() instanceof NoSuchObjectException) {
+ String methodName = method.getName();
+ if (!methodName.startsWith("get_table") && !methodName.startsWith("get_partition")) {
+ LOG.error(ExceptionUtils.getStackTrace(e.getCause()));
+ }
+ throw e.getCause();
} else if (e.getCause() instanceof MetaException && e.getCause().getCause() != null
&& e.getCause().getCause() instanceof javax.jdo.JDOException) {
// The JDOException may be wrapped further in a MetaException
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Context.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Context.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Context.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Context.java Tue Jul 9 09:07:35 2013
@@ -42,6 +42,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.TaskRunner;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.lockmgr.HiveLock;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager;
@@ -78,6 +79,7 @@ public class Context {
private final Configuration conf;
protected int pathid = 10000;
protected boolean explain = false;
+ protected boolean explainLogical = false;
protected String cmd = "";
// number of previous attempts
protected int tryCount = 0;
@@ -139,11 +141,26 @@ public class Context {
* Find whether the current query is an explain query
* @return true if the query is an explain query, false if not
*/
- public boolean getExplain () {
+ public boolean getExplain() {
return explain;
}
/**
+ * Find whether the current query is a logical explain query
+ */
+ public boolean getExplainLogical() {
+ return explainLogical;
+ }
+
+ /**
+ * Set the context on whether the current query is a logical
+ * explain query.
+ */
+ public void setExplainLogical(boolean explainLogical) {
+ this.explainLogical = explainLogical;
+ }
+
+ /**
* Set the original query command.
* @param cmd the original query command string
*/
@@ -165,16 +182,17 @@ public class Context {
* @param scheme Scheme of the target FS
* @param authority Authority of the target FS
* @param mkdir create the directory if true
- * @param scratchdir path of tmp directory
+ * @param scratchDir path of tmp directory
*/
private String getScratchDir(String scheme, String authority,
boolean mkdir, String scratchDir) {
String fileSystem = scheme + ":" + authority;
- String dir = fsScratchDirs.get(fileSystem);
+ String dir = fsScratchDirs.get(fileSystem + "-" + TaskRunner.getTaskRunnerID());
if (dir == null) {
- Path dirPath = new Path(scheme, authority, scratchDir);
+ Path dirPath = new Path(scheme, authority,
+ scratchDir + "-" + TaskRunner.getTaskRunnerID());
if (mkdir) {
try {
FileSystem fs = dirPath.getFileSystem(conf);
@@ -191,7 +209,7 @@ public class Context {
}
}
dir = dirPath.toString();
- fsScratchDirs.put(fileSystem, dir);
+ fsScratchDirs.put(fileSystem + "-" + TaskRunner.getTaskRunnerID(), dir);
}
return dir;
@@ -228,9 +246,10 @@ public class Context {
try {
Path dir = FileUtils.makeQualified(nonLocalScratchPath, conf);
URI uri = dir.toUri();
- return getScratchDir(uri.getScheme(), uri.getAuthority(),
+ String newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(),
!explain, uri.getPath());
-
+ LOG.info("New scratch dir is " + newScratchDir);
+ return newScratchDir;
} catch (IOException e) {
throw new RuntimeException(e);
} catch (IllegalArgumentException e) {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Tue Jul 9 09:07:35 2013
@@ -78,6 +78,9 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.metadata.formatting.JsonMetaDataFormatter;
+import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatUtils;
+import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatter;
import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.AbstractSemanticAnalyzerHook;
@@ -126,6 +129,7 @@ public class Driver implements CommandPr
private String errorMessage;
private String SQLState;
+ private Throwable downstreamError;
// A limit on the number of threads that can be launched
private int maxthreads;
@@ -143,6 +147,7 @@ public class Driver implements CommandPr
} catch (SemanticException e) {
errorMessage = "FAILED: Error in semantic analysis: " + e.getMessage();
SQLState = ErrorMsg.findSQLState(e.getMessage());
+ downstreamError = e;
console.printError(errorMessage, "\n"
+ org.apache.hadoop.util.StringUtils.stringifyException(e));
return false;
@@ -483,8 +488,9 @@ public class Driver implements CommandPr
perfLogger.PerfLogBegin(LOG, PerfLogger.DO_AUTHORIZATION);
doAuthorization(sem);
} catch (AuthorizationException authExp) {
- console.printError("Authorization failed:" + authExp.getMessage()
- + ". Use show grant to get more details.");
+ errorMessage = "Authorization failed:" + authExp.getMessage()
+ + ". Use show grant to get more details.";
+ console.printError(errorMessage);
return 403;
} finally {
perfLogger.PerfLogEnd(LOG, PerfLogger.DO_AUTHORIZATION);
@@ -502,6 +508,7 @@ public class Driver implements CommandPr
}
errorMessage += " " + e.getMessage();
SQLState = error.getSQLState();
+ downstreamError = e;
console.printError(errorMessage, "\n"
+ org.apache.hadoop.util.StringUtils.stringifyException(e));
return error.getErrorCode();
@@ -837,12 +844,14 @@ public class Driver implements CommandPr
} catch (SemanticException e) {
errorMessage = "FAILED: Error in acquiring locks: " + e.getMessage();
SQLState = ErrorMsg.findSQLState(e.getMessage());
+ downstreamError = e;
console.printError(errorMessage, "\n"
+ org.apache.hadoop.util.StringUtils.stringifyException(e));
return (10);
} catch (LockException e) {
errorMessage = "FAILED: Error in acquiring locks: " + e.getMessage();
SQLState = ErrorMsg.findSQLState(e.getMessage());
+ downstreamError = e;
console.printError(errorMessage, "\n"
+ org.apache.hadoop.util.StringUtils.stringifyException(e));
return (10);
@@ -869,8 +878,62 @@ public class Driver implements CommandPr
}
public CommandProcessorResponse run(String command) throws CommandNeedRetryException {
+ CommandProcessorResponse cpr = runInternal(command);
+ if(cpr.getResponseCode() == 0)
+ return cpr;
+ SessionState ss = SessionState.get();
+ if(ss == null)
+ return cpr;
+ MetaDataFormatter mdf = MetaDataFormatUtils.getFormatter(ss.getConf());
+ if(!(mdf instanceof JsonMetaDataFormatter))
+ return cpr;
+ /*Here we want to encode the error in machine readable way (e.g. JSON)
+ * Ideally, errorCode would always be set to a canonical error defined in ErrorMsg.
+ * In practice that is rarely the case, so the messy logic below tries to tease
+ * out canonical error code if it can. Exclude stack trace from output when
+ * the error is a specific/expected one.
+ * It's written to stdout for backward compatibility (WebHCat consumes it).*/
+ try {
+ if(downstreamError == null) {
+ mdf.error(ss.out, errorMessage, cpr.getResponseCode(), SQLState);
+ return cpr;
+ }
+ ErrorMsg canonicalErr = ErrorMsg.getErrorMsg(cpr.getResponseCode());
+ if(canonicalErr != null && canonicalErr != ErrorMsg.GENERIC_ERROR) {
+ /*Some HiveExceptions (e.g. SemanticException) don't set
+ canonical ErrorMsg explicitly, but there is logic
+ (e.g. #compile()) to find an appropriate canonical error and
+ return its code as error code. In this case we want to
+ preserve it for downstream code to interpret*/
+ mdf.error(ss.out, errorMessage, cpr.getResponseCode(), SQLState, null);
+ return cpr;
+ }
+ if(downstreamError instanceof HiveException) {
+ HiveException rc = (HiveException) downstreamError;
+ mdf.error(ss.out, errorMessage,
+ rc.getCanonicalErrorMsg().getErrorCode(), SQLState,
+ rc.getCanonicalErrorMsg() == ErrorMsg.GENERIC_ERROR ?
+ org.apache.hadoop.util.StringUtils.stringifyException(rc)
+ : null);
+ }
+ else {
+ ErrorMsg canonicalMsg =
+ ErrorMsg.getErrorMsg(downstreamError.getMessage());
+ mdf.error(ss.out, errorMessage, canonicalMsg.getErrorCode(),
+ SQLState, org.apache.hadoop.util.StringUtils.
+ stringifyException(downstreamError));
+ }
+ }
+ catch(HiveException ex) {
+ console.printError("Unable to JSON-encode the error",
+ org.apache.hadoop.util.StringUtils.stringifyException(ex));
+ }
+ return cpr;
+ }
+ private CommandProcessorResponse runInternal(String command) throws CommandNeedRetryException {
errorMessage = null;
SQLState = null;
+ downstreamError = null;
if (!validateConfVariables()) {
return new CommandProcessorResponse(12, errorMessage, SQLState);
@@ -885,10 +948,11 @@ public class Driver implements CommandPr
driverRunHook.preDriverRun(hookContext);
}
} catch (Exception e) {
- errorMessage = "FAILED: Hive Internal Error: " + Utilities.getNameMessage(e)
- + "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e);
+ errorMessage = "FAILED: Hive Internal Error: " + Utilities.getNameMessage(e);
SQLState = ErrorMsg.findSQLState(e.getMessage());
- console.printError(errorMessage);
+ downstreamError = e;
+ console.printError(errorMessage + "\n"
+ + org.apache.hadoop.util.StringUtils.stringifyException(e));
return new CommandProcessorResponse(12, errorMessage, SQLState);
}
@@ -961,10 +1025,11 @@ public class Driver implements CommandPr
driverRunHook.postDriverRun(hookContext);
}
} catch (Exception e) {
- errorMessage = "FAILED: Hive Internal Error: " + Utilities.getNameMessage(e)
- + "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e);
+ errorMessage = "FAILED: Hive Internal Error: " + Utilities.getNameMessage(e);
SQLState = ErrorMsg.findSQLState(e.getMessage());
- console.printError(errorMessage);
+ downstreamError = e;
+ console.printError(errorMessage + "\n"
+ + org.apache.hadoop.util.StringUtils.stringifyException(e));
return new CommandProcessorResponse(12, errorMessage, SQLState);
}
@@ -984,7 +1049,7 @@ public class Driver implements CommandPr
.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_UNION_REMOVE))))) {
errorMessage = "FAILED: Hive Internal Error: "
+ ErrorMsg.SUPPORT_DIR_MUST_TRUE_FOR_LIST_BUCKETING.getMsg();
- SQLState = ErrorMsg.findSQLState(errorMessage);
+ SQLState = ErrorMsg.SUPPORT_DIR_MUST_TRUE_FOR_LIST_BUCKETING.getSQLState();
console.printError(errorMessage + "\n");
valid = false;
}
@@ -1158,12 +1223,7 @@ public class Driver implements CommandPr
}
Task<? extends Serializable> backupTask = tsk.getAndInitBackupTask();
if (backupTask != null) {
- errorMessage = "FAILED: Execution Error, return code " + exitVal + " from "
- + tsk.getClass().getName();
- ErrorMsg em = ErrorMsg.getErrorMsg(exitVal);
- if (em != null) {
- errorMessage += ". " + em.getMsg();
- }
+ setErrorMsgAndDetail(exitVal, tskRes.getTaskError(), tsk);
console.printError(errorMessage);
errorMessage = "ATTEMPT: Execute BackupTask: " + backupTask.getClass().getName();
console.printError(errorMessage);
@@ -1184,13 +1244,7 @@ public class Driver implements CommandPr
perfLogger.PerfLogEnd(LOG, PerfLogger.FAILURE_HOOK + ofh.getClass().getName());
}
-
- errorMessage = "FAILED: Execution Error, return code " + exitVal + " from "
- + tsk.getClass().getName();
- ErrorMsg em = ErrorMsg.getErrorMsg(exitVal);
- if (em != null) {
- errorMessage += ". " + em.getMsg();
- }
+ setErrorMsgAndDetail(exitVal, tskRes.getTaskError(), tsk);
SQLState = "08S01";
console.printError(errorMessage);
if (!running.isEmpty()) {
@@ -1273,6 +1327,7 @@ public class Driver implements CommandPr
// TODO: do better with handling types of Exception here
errorMessage = "FAILED: Hive Internal Error: " + Utilities.getNameMessage(e);
SQLState = "08S01";
+ downstreamError = e;
console.printError(errorMessage + "\n"
+ org.apache.hadoop.util.StringUtils.stringifyException(e));
return (12);
@@ -1308,7 +1363,21 @@ public class Driver implements CommandPr
return (0);
}
-
+ private void setErrorMsgAndDetail(int exitVal, Throwable downstreamError, Task tsk) {
+ this.downstreamError = downstreamError;
+ errorMessage = "FAILED: Execution Error, return code " + exitVal + " from " + tsk.getClass().getName();
+ if(downstreamError != null) {
+ //here we assume that upstream code may have parametrized the msg from ErrorMsg
+ //so we want to keep it
+ errorMessage += ". " + downstreamError.getMessage();
+ }
+ else {
+ ErrorMsg em = ErrorMsg.getErrorMsg(exitVal);
+ if (em != null) {
+ errorMessage += ". " + em.getMsg();
+ }
+ }
+ }
/**
* Launches a new task
*
@@ -1388,7 +1457,7 @@ public class Driver implements CommandPr
while (true) {
while (resultIterator.hasNext()) {
TaskResult tskRes = resultIterator.next();
- if (tskRes.isRunning() == false) {
+ if (!tskRes.isRunning()) {
return tskRes;
}
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java Tue Jul 9 09:07:35 2013
@@ -32,11 +32,15 @@ import org.apache.hadoop.hive.ql.parse.A
/**
* List of all error messages.
* This list contains both compile time and run-time errors.
- **/
+ *
+ * This class supports parametrized messages such as (@link #TRUNCATE_FOR_NON_MANAGED_TABLE}. These are
+ * preferable over un-parametrized ones where arbitrary String is appended to the end of the message,
+ * for example {@link #getMsg(String)} and {@link #INVALID_TABLE}.
+ */
public enum ErrorMsg {
// The error codes are Hive-specific and partitioned into the following ranges:
- // 10000 to 19999: Errors occuring during semantic analysis and compilation of the query.
+ // 10000 to 19999: Errors occurring during semantic analysis and compilation of the query.
// 20000 to 29999: Runtime errors where Hive believes that retries are unlikely to succeed.
// 30000 to 39999: Runtime errors which Hive thinks may be transient and retrying may succeed.
// 40000 to 49999: Errors where Hive is unable to advise about retries.
@@ -239,7 +243,7 @@ public enum ErrorMsg {
JOINNODE_OUTERJOIN_MORETHAN_16(10142, "Single join node containing outer join(s) " +
"cannot have more than 16 aliases"),
- INVALID_JDO_FILTER_EXPRESSION(10043, "Invalid expression for JDO filter"),
+ INVALID_JDO_FILTER_EXPRESSION(10143, "Invalid expression for JDO filter"),
SHOW_CREATETABLE_INDEX(10144, "SHOW CREATE TABLE does not support tables of type INDEX_TABLE."),
ALTER_BUCKETNUM_NONBUCKETIZED_TBL(10145, "Table is not bucketized."),
@@ -321,13 +325,6 @@ public enum ErrorMsg {
"with distincts. Either set hive.new.job.grouping.set.cardinality to a high number " +
"(higher than the number of rows per input row due to grouping sets in the query), or " +
"rewrite the query to not use distincts."),
- TRUNCATE_COLUMN_INDEXED_TABLE(10227, "Can not truncate columns from table with indexes"),
- TRUNCATE_COLUMN_NOT_RC(10228, "Only RCFileFormat supports column truncation."),
- TRUNCATE_COLUMN_ARCHIVED(10229, "Column truncation cannot be performed on archived partitions."),
- TRUNCATE_BUCKETED_COLUMN(10230,
- "A column on which a partition/table is bucketed cannot be truncated."),
- TRUNCATE_LIST_BUCKETED_COLUMN(10231,
- "A column on which a partition/table is list bucketed cannot be truncated."),
OPERATOR_NOT_ALLOWED_WITH_MAPJOIN(10227,
"Not all clauses are supported with mapjoin hint. Please remove mapjoin hint."),
@@ -350,6 +347,20 @@ public enum ErrorMsg {
TABLES_INCOMPATIBLE_SCHEMAS(10235, "Tables have incompatible schemas and their partitions " +
" cannot be exchanged."),
+ TRUNCATE_COLUMN_INDEXED_TABLE(10236, "Can not truncate columns from table with indexes"),
+ TRUNCATE_COLUMN_NOT_RC(10237, "Only RCFileFormat supports column truncation."),
+ TRUNCATE_COLUMN_ARCHIVED(10238, "Column truncation cannot be performed on archived partitions."),
+ TRUNCATE_BUCKETED_COLUMN(10239,
+ "A column on which a partition/table is bucketed cannot be truncated."),
+ TRUNCATE_LIST_BUCKETED_COLUMN(10240,
+ "A column on which a partition/table is list bucketed cannot be truncated."),
+
+ TABLE_NOT_PARTITIONED(10241, "Table {0} is not a partitioned table", true),
+ DATABSAE_ALREADY_EXISTS(10242, "Database {0} already exists", true),
+ CANNOT_REPLACE_COLUMNS(10243, "Replace columns is not supported for table {0}. SerDe may be incompatible.", true),
+ BAD_LOCATION_VALUE(10244, "{0} is not absolute or has no scheme information. Please specify a complete absolute uri with scheme information."),
+ UNSUPPORTED_ALTER_TBL_OP(10245, "{0} alter table options is not supported"),
+
SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."),
SCRIPT_IO_ERROR(20001, "An error occurred while reading or writing to your custom script. "
+ "It may have crashed with an error."),
@@ -487,7 +498,7 @@ public enum ErrorMsg {
/**
* For a given error message string, searches for a <code>ErrorMsg</code> enum
- * that appears to be a match. If an match is found, returns the
+ * that appears to be a match. If a match is found, returns the
* <code>SQLState</code> associated with the <code>ErrorMsg</code>. If a match
* is not found or <code>ErrorMsg</code> has no <code>SQLState</code>, returns
* the <code>SQLState</code> bound to the <code>GENERIC_ERROR</code>
@@ -604,10 +615,41 @@ public enum ErrorMsg {
public String format(String reason) {
return format(new String[]{reason});
}
-
+ /**
+ * If the message is parametrized, this will fill the parameters with supplied
+ * {@code reasons}, otherwise {@code reasons} are appended at the end of the
+ * message.
+ */
public String format(String... reasons) {
- assert format != null;
- return format.format(reasons);
+ /* Not all messages are parametrized even those that should have been, e.g {@link #INVALID_TABLE}.
+ INVALID_TABLE is usually used with {@link #getMsg(String)}.
+ This method can also be used with INVALID_TABLE and the like and will match getMsg(String) behavior.
+
+ Another example: {@link #INVALID_PARTITION}. Ideally you want the message to have 2 parameters one for
+ partition name one for table name. Since this is already defined w/o any parameters, one can still call
+ {@code INVALID_PARTITION.format("<partName> <table Name>"}. This way the message text will be slightly
+ different but at least the errorCode will match. Note this, should not be abused by adding anything other
+ than what should have been parameter names to keep msg text standardized.
+ */
+ if(reasons == null || reasons.length == 0) {
+ return getMsg();
+ }
+ if(format != null) {
+ return format.format(reasons);
+ }
+ if(reasons.length > 1) {
+ StringBuilder sb = new StringBuilder();
+ for(String re : reasons) {
+ if(re != null) {
+ if(sb.length() > 0) {
+ sb.append(" ");
+ }
+ sb.append(re);
+ }
+ }
+ return getMsg(sb.toString());
+ }
+ return getMsg(reasons[0]);
}
public String getErrorCodedMsg() {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java Tue Jul 9 09:07:35 2013
@@ -390,17 +390,4 @@ public class ColumnStatsTask extends Tas
public String getName() {
return "COLUMNSTATS TASK";
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- FetchWork fWork = work.getfWork();
- String s = fWork.getTblDir();
- if ((s != null) && ctx.isMRTmpFileURI(s)) {
- fWork.setTblDir(ctx.localizeMRTmpFileURI(s));
- }
- ArrayList<String> ls = fWork.getPartDir();
- if (ls != null) {
- ctx.localizePaths(ls);
- }
- }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java Tue Jul 9 09:07:35 2013
@@ -360,26 +360,46 @@ public abstract class CommonJoinOperator
// filter tags for objects
protected transient short[] filterTags;
- // ANDed value of all filter tags in current join group
- // if any of values passes on outer join alias (which makes zero for the tag alias),
- // it means there exists a pair for it, and no need to check outer join (just do inner join)
- //
- // for example, with table a, b something like,
- // a, b = 100, 10 | 100, 20 | 100, 30
- //
- // the query "a FOJ b ON a.k=b.k AND a.v>0 AND b.v>20" makes values with tag
- //
- // a = 100, 10, 00000010 | 100, 20, 00000010 | 100, 30, 00000010 : 0/1 for 'b' (alias 1)
- // b = 100, 10, 00000001 | 100, 20, 00000001 | 100, 30, 00000000 : 0/1 for 'a' (alias 0)
- //
- // which makes aliasFilterTags for a = 00000010, for b = 00000000
- //
- // for LO, b = 0000000(0) means there is a pair object(s) in 'b' (has no 'a'-null case)
- // for RO, a = 000000(1)0 means there is no pair object in 'a' (has null-'b' case)
- //
- // result : 100, 10 + 100, 30 | 100, 20 + 100, 30 | 100, 30 + 100, 30 |
- // N + 100, 10 | N + 100, 20
- //
+ /**
+ * On filterTags
+ *
+ * ANDed value of all filter tags in current join group
+ * if any of values passes on outer join alias (which makes zero for the tag alias),
+ * it means there exists a pair for it and safely regarded as a inner join
+ *
+ * for example, with table a, b something like,
+ * a = 100, 10 | 100, 20 | 100, 30
+ * b = 100, 10 | 100, 20 | 100, 30
+ *
+ * the query "a FO b ON a.k=b.k AND a.v>10 AND b.v>30" makes filter map
+ * 0(a) = [1(b),1] : a.v>10
+ * 1(b) = [0(a),1] : b.v>30
+ *
+ * for filtered rows in a (100,10) create a-NULL
+ * for filtered rows in b (100,10) (100,20) (100,30) create NULL-b
+ *
+ * with 0(a) = [1(b),1] : a.v>10
+ * 100, 10 = 00000010 (filtered)
+ * 100, 20 = 00000000 (valid)
+ * 100, 30 = 00000000 (valid)
+ * -------------------------
+ * sum = 00000000 : for valid rows in b, there is at least one pair in a
+ *
+ * with 1(b) = [0(a),1] : b.v>30
+ * 100, 10 = 00000001 (filtered)
+ * 100, 20 = 00000001 (filtered)
+ * 100, 30 = 00000001 (filtered)
+ * -------------------------
+ * sum = 00000001 : for valid rows in a (100,20) (100,30), there is no pair in b
+ *
+ * result :
+ * 100, 10 : N, N
+ * N, N : 100, 10
+ * N, N : 100, 20
+ * N, N : 100, 30
+ * 100, 20 : N, N
+ * 100, 30 : N, N
+ */
protected transient short[] aliasFilterTags;
// all evaluation should be processed here for valid aliasFilterTags
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java Tue Jul 9 09:07:35 2013
@@ -205,15 +205,6 @@ public class ConditionalTask extends Tas
}
@Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- if (getListTasks() != null) {
- for (Task<? extends Serializable> t : getListTasks()) {
- t.localizeMRTmpFiles(ctx);
- }
- }
- }
-
- @Override
public List<Task<? extends Serializable>> getDependentTasks() {
return listTasks;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java Tue Jul 9 09:07:35 2013
@@ -106,12 +106,4 @@ public class CopyTask extends Task<CopyW
public String getName() {
return "COPY";
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // copy task is only used by the load command and
- // does not use any map-reduce tmp files
- // we don't expect to enter this code path at all
- throw new RuntimeException ("Unexpected call");
- }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Tue Jul 9 09:07:35 2013
@@ -80,6 +80,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.exec.ArchiveUtils.PartSpecInfo;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
@@ -102,10 +103,8 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.metadata.formatting.JsonMetaDataFormatter;
import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatUtils;
import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatter;
-import org.apache.hadoop.hive.ql.metadata.formatting.TextMetaDataFormatter;
import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
@@ -205,14 +204,7 @@ public class DDLTask extends Task<DDLWor
// Pick the formatter to use to display the results. Either the
// normal human readable output or a json object.
- if ("json".equals(conf.get(
- HiveConf.ConfVars.HIVE_DDL_OUTPUT_FORMAT.varname, "text"))) {
- formatter = new JsonMetaDataFormatter();
- } else {
- formatter = new TextMetaDataFormatter(
- conf.getIntVar(HiveConf.ConfVars.CLIPRETTYOUTPUTNUMCOLS));
- }
-
+ formatter = MetaDataFormatUtils.getFormatter(conf);
INTERMEDIATE_ARCHIVED_DIR_SUFFIX =
HiveConf.getVar(conf, ConfVars.METASTORE_INT_ARCHIVED);
INTERMEDIATE_ORIGINAL_DIR_SUFFIX =
@@ -437,32 +429,10 @@ public class DDLTask extends Task<DDLWor
if (alterTableExchangePartition != null) {
return exchangeTablePartition(db, alterTableExchangePartition);
}
-
- } catch (InvalidTableException e) {
- formatter.consoleError(console, "Table " + e.getTableName() + " does not exist",
- formatter.MISSING);
- LOG.debug(stringifyException(e));
- return 1;
- } catch (AlreadyExistsException e) {
- formatter.consoleError(console, e.getMessage(), formatter.CONFLICT);
- return 1;
- } catch (NoSuchObjectException e) {
- formatter.consoleError(console, e.getMessage(),
- "\n" + stringifyException(e),
- formatter.MISSING);
- return 1;
- } catch (HiveException e) {
- formatter.consoleError(console,
- "FAILED: Error in metadata: " + e.getMessage(),
- "\n" + stringifyException(e),
- formatter.ERROR);
- LOG.debug(stringifyException(e));
+ } catch (Throwable e) {
+ setException(e);
+ LOG.error(stringifyException(e));
return 1;
- } catch (Exception e) {
- formatter.consoleError(console, "Failed with exception " + e.getMessage(),
- "\n" + stringifyException(e),
- formatter.ERROR);
- return (1);
}
assert false;
return 0;
@@ -881,7 +851,7 @@ public class DDLTask extends Task<DDLWor
}
db.alterDatabase(database.getName(), database);
} else {
- throw new HiveException("ERROR: The database " + dbName + " does not exist.");
+ throw new HiveException(ErrorMsg.DATABASE_NOT_EXISTS, dbName);
}
return 0;
}
@@ -1000,16 +970,6 @@ public class DDLTask extends Task<DDLWor
* @return Returns 0 when execution succeeds and above 0 if it fails.
* @throws HiveException
*/
- /**
- * Add a partition to a table.
- *
- * @param db
- * Database to add the partition to.
- * @param addPartitionDesc
- * Add this partition.
- * @return Returns 0 when execution succeeds and above 0 if it fails.
- * @throws HiveException
- */
private int addPartition(Hive db, AddPartitionDesc addPartitionDesc) throws HiveException {
Table tbl = db.getTable(addPartitionDesc.getDbName(), addPartitionDesc.getTableName());
@@ -1118,7 +1078,7 @@ public class DDLTask extends Task<DDLWor
try {
db.alterTable(tabName, tbl);
} catch (InvalidOperationException e) {
- throw new HiveException("Uable to update table");
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "Unable to alter " + tabName);
}
work.getInputs().add(new ReadEntity(tbl));
@@ -1878,10 +1838,7 @@ public class DDLTask extends Task<DDLWor
tbl = db.getTable(tabName);
if (!tbl.isPartitioned()) {
- formatter.consoleError(console,
- "Table " + tabName + " is not a partitioned table",
- formatter.ERROR);
- return 1;
+ throw new HiveException(ErrorMsg.TABLE_NOT_PARTITIONED, tabName);
}
if (showParts.getPartSpec() != null) {
parts = db.getPartitionNames(tbl.getDbName(),
@@ -1899,20 +1856,12 @@ public class DDLTask extends Task<DDLWor
formatter.showTablePartitons(outStream, parts);
- ((FSDataOutputStream) outStream).close();
+ outStream.close();
outStream = null;
- } catch (FileNotFoundException e) {
- formatter.logWarn(outStream, "show partitions: " + stringifyException(e),
- MetaDataFormatter.ERROR);
- return 1;
- } catch (IOException e) {
- formatter.logWarn(outStream, "show partitions: " + stringifyException(e),
- MetaDataFormatter.ERROR);
- return 1;
} catch (Exception e) {
- throw new HiveException(e);
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "show partitions for table " + tabName);
} finally {
- IOUtils.closeStream((FSDataOutputStream) outStream);
+ IOUtils.closeStream(outStream);
}
return 0;
@@ -2212,7 +2161,7 @@ public class DDLTask extends Task<DDLWor
/**
* Write a list of the available databases to a file.
*
- * @param showDatabases
+ * @param showDatabasesDesc
* These are the databases we're interested in.
* @return Returns 0 when execution succeeds and above 0 if it fails.
* @throws HiveException
@@ -2237,20 +2186,12 @@ public class DDLTask extends Task<DDLWor
outStream = fs.create(resFile);
formatter.showDatabases(outStream, databases);
- ((FSDataOutputStream) outStream).close();
+ outStream.close();
outStream = null;
- } catch (FileNotFoundException e) {
- formatter.logWarn(outStream, "show databases: " + stringifyException(e),
- formatter.ERROR);
- return 1;
- } catch (IOException e) {
- formatter.logWarn(outStream, "show databases: " + stringifyException(e),
- formatter.ERROR);
- return 1;
} catch (Exception e) {
- throw new HiveException(e.toString());
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "show databases");
} finally {
- IOUtils.closeStream((FSDataOutputStream) outStream);
+ IOUtils.closeStream(outStream);
}
return 0;
}
@@ -2272,8 +2213,7 @@ public class DDLTask extends Task<DDLWor
String dbName = showTbls.getDbName();
if (!db.databaseExists(dbName)) {
- throw new HiveException("ERROR: The database " + dbName + " does not exist.");
-
+ throw new HiveException(ErrorMsg.DATABASE_NOT_EXISTS, dbName);
}
if (showTbls.getPattern() != null) {
LOG.info("pattern: " + showTbls.getPattern());
@@ -2292,20 +2232,12 @@ public class DDLTask extends Task<DDLWor
SortedSet<String> sortedTbls = new TreeSet<String>(tbls);
formatter.showTables(outStream, sortedTbls);
- ((FSDataOutputStream) outStream).close();
+ outStream.close();
outStream = null;
- } catch (FileNotFoundException e) {
- formatter.logWarn(outStream, "show table: " + stringifyException(e),
- formatter.ERROR);
- return 1;
- } catch (IOException e) {
- formatter.logWarn(outStream, "show table: " + stringifyException(e),
- formatter.ERROR);
- return 1;
} catch (Exception e) {
- throw new HiveException(e.toString());
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "in database" + dbName);
} finally {
- IOUtils.closeStream((FSDataOutputStream) outStream);
+ IOUtils.closeStream(outStream);
}
return 0;
}
@@ -2337,8 +2269,7 @@ public class DDLTask extends Task<DDLWor
((FSDataOutputStream) outStream).close();
outStream = null;
} catch (IOException e) {
- LOG.warn("show columns: " + stringifyException(e));
- return 1;
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR);
} finally {
IOUtils.closeStream((FSDataOutputStream) outStream);
}
@@ -2668,9 +2599,7 @@ public class DDLTask extends Task<DDLWor
Database database = db.getDatabase(descDatabase.getDatabaseName());
if (database == null) {
- formatter.error(outStream,
- "No such database: " + descDatabase.getDatabaseName(),
- formatter.MISSING);
+ throw new HiveException(ErrorMsg.DATABASE_NOT_EXISTS, descDatabase.getDatabaseName());
} else {
Map<String, String> params = null;
if(descDatabase.isExt()) {
@@ -2683,22 +2612,12 @@ public class DDLTask extends Task<DDLWor
database.getLocationUri(),
params);
}
- ((FSDataOutputStream) outStream).close();
+ outStream.close();
outStream = null;
- } catch (FileNotFoundException e) {
- formatter.logWarn(outStream,
- "describe database: " + stringifyException(e),
- formatter.ERROR);
- return 1;
} catch (IOException e) {
- formatter.logWarn(outStream,
- "describe database: " + stringifyException(e),
- formatter.ERROR);
- return 1;
- } catch (Exception e) {
- throw new HiveException(e.toString());
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR);
} finally {
- IOUtils.closeStream((FSDataOutputStream) outStream);
+ IOUtils.closeStream(outStream);
}
return 0;
}
@@ -2713,7 +2632,7 @@ public class DDLTask extends Task<DDLWor
* @return Return 0 when execution succeeds and above 0 if it fails.
*/
private int showTableStatus(Hive db, ShowTableStatusDesc showTblStatus) throws HiveException {
- // get the tables for the desired pattenn - populate the output stream
+ // get the tables for the desired pattern - populate the output stream
List<Table> tbls = new ArrayList<Table>();
Map<String, String> part = showTblStatus.getPartSpec();
Partition par = null;
@@ -2749,20 +2668,12 @@ public class DDLTask extends Task<DDLWor
formatter.showTableStatus(outStream, db, conf, tbls, part, par);
- ((FSDataOutputStream) outStream).close();
+ outStream.close();
outStream = null;
- } catch (FileNotFoundException e) {
- formatter.logInfo(outStream, "show table status: " + stringifyException(e),
- formatter.ERROR);
- return 1;
- } catch (IOException e) {
- formatter.logInfo(outStream, "show table status: " + stringifyException(e),
- formatter.ERROR);
- return 1;
} catch (Exception e) {
- throw new HiveException(e);
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "show table status");
} finally {
- IOUtils.closeStream((FSDataOutputStream) outStream);
+ IOUtils.closeStream(outStream);
}
return 0;
}
@@ -2835,7 +2746,6 @@ public class DDLTask extends Task<DDLWor
return 0;
}
-
/**
* Write the description of a table to a file.
*
@@ -2860,36 +2770,26 @@ public class DDLTask extends Task<DDLWor
if (tbl == null) {
FileSystem fs = resFile.getFileSystem(conf);
outStream = fs.create(resFile);
- String errMsg = "Table " + tableName + " does not exist";
- formatter.error(outStream, errMsg, formatter.MISSING);
- ((FSDataOutputStream) outStream).close();
+ outStream.close();
outStream = null;
- return 0;
+ throw new HiveException(ErrorMsg.INVALID_TABLE, tableName);
}
if (descTbl.getPartSpec() != null) {
part = db.getPartition(tbl, descTbl.getPartSpec(), false);
if (part == null) {
FileSystem fs = resFile.getFileSystem(conf);
outStream = fs.create(resFile);
- String errMsg = "Partition " + descTbl.getPartSpec() + " for table "
- + tableName + " does not exist";
- formatter.error(outStream, errMsg, formatter.MISSING);
- ((FSDataOutputStream) outStream).close();
+ outStream.close();
outStream = null;
- return 0;
+ throw new HiveException(ErrorMsg.INVALID_PARTITION,
+ StringUtils.join(descTbl.getPartSpec().keySet(), ','), tableName);
}
tbl = part.getTable();
}
- } catch (FileNotFoundException e) {
- formatter.logInfo(outStream, "describe table: " + stringifyException(e),
- formatter.ERROR);
- return 1;
} catch (IOException e) {
- formatter.logInfo(outStream, "describe table: " + stringifyException(e),
- formatter.ERROR);
- return 1;
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR, tableName);
} finally {
- IOUtils.closeStream((FSDataOutputStream) outStream);
+ IOUtils.closeStream(outStream);
}
try {
@@ -2916,21 +2816,13 @@ public class DDLTask extends Task<DDLWor
descTbl.isFormatted(), descTbl.isExt(), descTbl.isPretty());
LOG.info("DDLTask: written data for " + tbl.getTableName());
- ((FSDataOutputStream) outStream).close();
+ outStream.close();
outStream = null;
- } catch (FileNotFoundException e) {
- formatter.logInfo(outStream, "describe table: " + stringifyException(e),
- formatter.ERROR);
- return 1;
} catch (IOException e) {
- formatter.logInfo(outStream, "describe table: " + stringifyException(e),
- formatter.ERROR);
- return 1;
- } catch (Exception e) {
- throw new HiveException(e);
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR, tableName);
} finally {
- IOUtils.closeStream((FSDataOutputStream) outStream);
+ IOUtils.closeStream(outStream);
}
return 0;
@@ -3009,11 +2901,8 @@ public class DDLTask extends Task<DDLWor
if (alterTbl.getOp() != AlterTableDesc.AlterTableTypes.ALTERPROTECTMODE) {
part = db.getPartition(tbl, alterTbl.getPartSpec(), false);
if (part == null) {
- formatter.consoleError(console,
- "Partition : " + alterTbl.getPartSpec().toString()
- + " does not exist.",
- formatter.MISSING);
- return 1;
+ throw new HiveException(ErrorMsg.INVALID_PARTITION,
+ StringUtils.join(alterTbl.getPartSpec().keySet(), ',') + " for table " + alterTbl.getOldName());
}
}
else {
@@ -3044,10 +2933,7 @@ public class DDLTask extends Task<DDLWor
while (iterOldCols.hasNext()) {
String oldColName = iterOldCols.next().getName();
if (oldColName.equalsIgnoreCase(newColName)) {
- formatter.consoleError(console,
- "Column '" + newColName + "' exists",
- formatter.CONFLICT);
- return 1;
+ throw new HiveException(ErrorMsg.DUPLICATE_COLUMN_NAMES, newColName);
}
}
oldCols.add(newCol);
@@ -3078,10 +2964,7 @@ public class DDLTask extends Task<DDLWor
String oldColName = col.getName();
if (oldColName.equalsIgnoreCase(newName)
&& !oldColName.equalsIgnoreCase(oldName)) {
- formatter.consoleError(console,
- "Column '" + newName + "' exists",
- formatter.CONFLICT);
- return 1;
+ throw new HiveException(ErrorMsg.DUPLICATE_COLUMN_NAMES, newName);
} else if (oldColName.equalsIgnoreCase(oldName)) {
col.setName(newName);
if (type != null && !type.trim().equals("")) {
@@ -3108,17 +2991,11 @@ public class DDLTask extends Task<DDLWor
// did not find the column
if (!found) {
- formatter.consoleError(console,
- "Column '" + oldName + "' does not exists",
- formatter.MISSING);
- return 1;
+ throw new HiveException(ErrorMsg.INVALID_COLUMN, oldName);
}
// after column is not null, but we did not find it.
if ((afterCol != null && !afterCol.trim().equals("")) && position < 0) {
- formatter.consoleError(console,
- "Column '" + afterCol + "' does not exists",
- formatter.MISSING);
- return 1;
+ throw new HiveException(ErrorMsg.INVALID_COLUMN, afterCol);
}
if (position >= 0) {
@@ -3139,11 +3016,7 @@ public class DDLTask extends Task<DDLWor
&& !tbl.getSerializationLib().equals(LazySimpleSerDe.class.getName())
&& !tbl.getSerializationLib().equals(ColumnarSerDe.class.getName())
&& !tbl.getSerializationLib().equals(DynamicSerDe.class.getName())) {
- formatter.consoleError(console,
- "Replace columns is not supported for this table. "
- + "SerDe may be incompatible.",
- formatter.ERROR);
- return 1;
+ throw new HiveException(ErrorMsg.CANNOT_REPLACE_COLUMNS, alterTbl.getOldName());
}
tbl.getTTable().getSd().setCols(alterTbl.getNewCols());
} else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDPROPS) {
@@ -3242,10 +3115,7 @@ public class DDLTask extends Task<DDLWor
URI locUri = new URI(newLocation);
if (!locUri.isAbsolute() || locUri.getScheme() == null
|| locUri.getScheme().trim().equals("")) {
- throw new HiveException(
- newLocation
- + " is not absolute or has no scheme information. "
- + "Please specify a complete absolute uri with scheme information.");
+ throw new HiveException(ErrorMsg.BAD_LOCATION_VALUE, newLocation);
}
if (part != null) {
part.setLocation(newLocation);
@@ -3313,34 +3183,18 @@ public class DDLTask extends Task<DDLWor
tbl.setNumBuckets(alterTbl.getNumberBuckets());
}
} else {
- formatter.consoleError(console,
- "Unsupported Alter commnad",
- formatter.ERROR);
- return 1;
+ throw new HiveException(ErrorMsg.UNSUPPORTED_ALTER_TBL_OP, alterTbl.getOp().toString());
}
if (part == null && allPartitions == null) {
- if (!updateModifiedParameters(tbl.getTTable().getParameters(), conf)) {
- return 1;
- }
- try {
- tbl.checkValidity();
- } catch (HiveException e) {
- formatter.consoleError(console,
- "Invalid table columns : " + e.getMessage(),
- formatter.ERROR);
- return 1;
- }
+ updateModifiedParameters(tbl.getTTable().getParameters(), conf);
+ tbl.checkValidity();
} else if (part != null) {
- if (!updateModifiedParameters(part.getParameters(), conf)) {
- return 1;
- }
+ updateModifiedParameters(part.getParameters(), conf);
}
else {
for (Partition tmpPart: allPartitions) {
- if (!updateModifiedParameters(tmpPart.getParameters(), conf)) {
- return 1;
- }
+ updateModifiedParameters(tmpPart.getParameters(), conf);
}
}
@@ -3354,11 +3208,8 @@ public class DDLTask extends Task<DDLWor
db.alterPartitions(tbl.getTableName(), allPartitions);
}
} catch (InvalidOperationException e) {
- console.printError("Invalid alter operation: " + e.getMessage());
LOG.info("alter table: " + stringifyException(e));
- return 1;
- } catch (HiveException e) {
- return 1;
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR);
}
// This is kind of hacky - the read entity contains the old table, whereas
@@ -3525,16 +3376,12 @@ public class DDLTask extends Task<DDLWor
* @param user
* user that is doing the updating.
*/
- private boolean updateModifiedParameters(Map<String, String> params, HiveConf conf) {
+ private boolean updateModifiedParameters(Map<String, String> params, HiveConf conf) throws HiveException {
String user = null;
try {
user = conf.getUser();
} catch (IOException e) {
- formatter.consoleError(console,
- "Unable to get current user: " + e.getMessage(),
- stringifyException(e),
- formatter.ERROR);
- return false;
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "Unable to get current user");
}
params.put("last_modified_by", user);
@@ -3562,17 +3409,21 @@ public class DDLTask extends Task<DDLWor
* @param crtDb
* @return Always returns 0
* @throws HiveException
- * @throws AlreadyExistsException
*/
private int createDatabase(Hive db, CreateDatabaseDesc crtDb)
- throws HiveException, AlreadyExistsException {
+ throws HiveException {
Database database = new Database();
database.setName(crtDb.getName());
database.setDescription(crtDb.getComment());
database.setLocationUri(crtDb.getLocationUri());
database.setParameters(crtDb.getDatabaseProperties());
-
- db.createDatabase(database, crtDb.getIfNotExists());
+ try {
+ db.createDatabase(database, crtDb.getIfNotExists());
+ }
+ catch (AlreadyExistsException ex) {
+ //it would be better if AlreadyExistsException had an errorCode field....
+ throw new HiveException(ex, ErrorMsg.DATABSAE_ALREADY_EXISTS, crtDb.getName());
+ }
return 0;
}
@@ -3582,11 +3433,15 @@ public class DDLTask extends Task<DDLWor
* @param dropDb
* @return Always returns 0
* @throws HiveException
- * @throws NoSuchObjectException
*/
private int dropDatabase(Hive db, DropDatabaseDesc dropDb)
- throws HiveException, NoSuchObjectException {
- db.dropDatabase(dropDb.getDatabaseName(), true, dropDb.getIfExists(), dropDb.isCasdade());
+ throws HiveException {
+ try {
+ db.dropDatabase(dropDb.getDatabaseName(), true, dropDb.getIfExists(), dropDb.isCasdade());
+ }
+ catch (NoSuchObjectException ex) {
+ throw new HiveException(ex, ErrorMsg.DATABASE_NOT_EXISTS, dropDb.getDatabaseName());
+ }
return 0;
}
@@ -3601,7 +3456,7 @@ public class DDLTask extends Task<DDLWor
throws HiveException {
String dbName = switchDb.getDatabaseName();
if (!db.databaseExists(dbName)) {
- throw new HiveException("ERROR: The database " + dbName + " does not exist.");
+ throw new HiveException(ErrorMsg.DATABASE_NOT_EXISTS, dbName);
}
db.setCurrentDatabase(dbName);
@@ -3988,7 +3843,7 @@ public class DDLTask extends Task<DDLWor
fs.mkdirs(location);
}
} catch (Exception e) {
- throw new HiveException(e);
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR);
}
return 0;
}
@@ -4023,15 +3878,11 @@ public class DDLTask extends Task<DDLWor
return locations;
}
- private int setGenericTableAttributes(Table tbl) {
+ private int setGenericTableAttributes(Table tbl) throws HiveException {
try {
tbl.setOwner(conf.getUser());
} catch (IOException e) {
- formatter.consoleError(console,
- "Unable to get current user: " + e.getMessage(),
- stringifyException(e),
- formatter.ERROR);
- return 1;
+ throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "Unable to get current user");
}
// set create time
tbl.setCreateTime((int) (System.currentTimeMillis() / 1000));
@@ -4059,9 +3910,4 @@ public class DDLTask extends Task<DDLWor
public String getName() {
return "DDL";
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // no-op
- }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java Tue Jul 9 09:07:35 2013
@@ -53,12 +53,4 @@ public class DependencyCollectionTask ex
public String getName() {
return "DEPENDENCY_COLLECTION";
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // copy task doesn't have any execution and so
- // does not use any map-reduce tmp files
- // we don't expect to enter this code path at all
- throw new RuntimeException ("Unexpected call");
- }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Tue Jul 9 09:07:35 2013
@@ -79,6 +79,7 @@ import org.apache.hadoop.hive.ql.session
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.ql.stats.StatsFactory;
import org.apache.hadoop.hive.ql.stats.StatsPublisher;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
@@ -89,6 +90,7 @@ import org.apache.hadoop.mapred.JobClien
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Partitioner;
import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.mapred.lib.TotalOrderPartitioner;
import org.apache.log4j.Appender;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.FileAppender;
@@ -437,6 +439,18 @@ public class ExecDriver extends Task<Map
addInputPaths(job, work, emptyScratchDirStr, ctx);
Utilities.setMapRedWork(job, work, ctx.getMRTmpFileURI());
+
+ if (work.getSamplingType() > 0 && work.getNumReduceTasks() > 1) {
+ try {
+ handleSampling(driverContext, work, job, new HiveConf(conf));
+ job.setPartitionerClass(HiveTotalOrderPartitioner.class);
+ } catch (Exception e) {
+ console.printInfo("Not enough sampling data.. Rolling back to single reducer task");
+ work.setNumReduceTasks(1);
+ job.setNumReduceTasks(1);
+ }
+ }
+
// remove the pwd from conf file so that job tracker doesn't show this
// logs
String pwd = HiveConf.getVar(job, HiveConf.ConfVars.METASTOREPWD);
@@ -585,6 +599,64 @@ public class ExecDriver extends Task<Map
}
}
+ private void handleSampling(DriverContext context, MapredWork work, JobConf job, HiveConf conf)
+ throws Exception {
+ assert work.getAliasToWork().keySet().size() == 1;
+
+ String alias = work.getAliases().get(0);
+ Operator<?> topOp = work.getAliasToWork().get(alias);
+ PartitionDesc partDesc = work.getAliasToPartnInfo().get(alias);
+
+ ArrayList<String> paths = work.getPaths();
+ ArrayList<PartitionDesc> parts = work.getPartitionDescs();
+
+ Path onePath = new Path(paths.get(0));
+ String tmpPath = context.getCtx().getExternalTmpFileURI(onePath.toUri());
+
+ Path partitionFile = new Path(tmpPath, ".partitions");
+ TotalOrderPartitioner.setPartitionFile(job, partitionFile);
+
+ PartitionKeySampler sampler = new PartitionKeySampler();
+
+ if (work.getSamplingType() == MapredWork.SAMPLING_ON_PREV_MR) {
+ console.printInfo("Use sampling data created in previous MR");
+ // merges sampling data from previous MR and make paritition keys for total sort
+ for (String path : paths) {
+ Path inputPath = new Path(path);
+ FileSystem fs = inputPath.getFileSystem(job);
+ for (FileStatus status : fs.globStatus(new Path(inputPath, ".sampling*"))) {
+ sampler.addSampleFile(status.getPath(), job);
+ }
+ }
+ } else if (work.getSamplingType() == MapredWork.SAMPLING_ON_START) {
+ console.printInfo("Creating sampling data..");
+ assert topOp instanceof TableScanOperator;
+ TableScanOperator ts = (TableScanOperator) topOp;
+
+ FetchWork fetchWork;
+ if (!partDesc.isPartitioned()) {
+ assert paths.size() == 1;
+ fetchWork = new FetchWork(paths.get(0), partDesc.getTableDesc());
+ } else {
+ fetchWork = new FetchWork(paths, parts, partDesc.getTableDesc());
+ }
+ fetchWork.setSource(ts);
+
+ // random sampling
+ FetchOperator fetcher = PartitionKeySampler.createSampler(fetchWork, conf, job, ts);
+ try {
+ ts.initialize(conf, new ObjectInspector[]{fetcher.getOutputObjectInspector()});
+ ts.setOutputCollector(sampler);
+ while (fetcher.pushRow()) { }
+ } finally {
+ fetcher.clearFetchContext();
+ }
+ } else {
+ throw new IllegalArgumentException("Invalid sampling type " + work.getSamplingType());
+ }
+ sampler.writePartitionKeys(partitionFile, job);
+ }
+
/**
* Set hive input format, and input format file if necessary.
*/
@@ -989,57 +1061,6 @@ public class ExecDriver extends Task<Map
}
@Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
-
- // localize any map-reduce input paths
- ctx.localizeKeys((Map<String, Object>) ((Object) work.getPathToAliases()));
- ctx.localizeKeys((Map<String, Object>) ((Object) work.getPathToPartitionInfo()));
-
- // localize any input paths for maplocal work
- MapredLocalWork l = work.getMapLocalWork();
- if (l != null) {
- Map<String, FetchWork> m = l.getAliasToFetchWork();
- if (m != null) {
- for (FetchWork fw : m.values()) {
- String s = fw.getTblDir();
- if ((s != null) && ctx.isMRTmpFileURI(s)) {
- fw.setTblDir(ctx.localizeMRTmpFileURI(s));
- }
- }
- }
- }
-
- // fix up outputs
- Map<String, ArrayList<String>> pa = work.getPathToAliases();
- if (pa != null) {
- for (List<String> ls : pa.values()) {
- for (String a : ls) {
- ArrayList<Operator<? extends OperatorDesc>> opList =
- new ArrayList<Operator<? extends OperatorDesc>>();
- opList.add(work.getAliasToWork().get(a));
-
- while (!opList.isEmpty()) {
- Operator<? extends OperatorDesc> op = opList.remove(0);
-
- if (op instanceof FileSinkOperator) {
- FileSinkDesc fdesc = ((FileSinkOperator) op).getConf();
- String s = fdesc.getDirName();
- if ((s != null) && ctx.isMRTmpFileURI(s)) {
- fdesc.setDirName(ctx.localizeMRTmpFileURI(s));
- }
- ((FileSinkOperator) op).setConf(fdesc);
- }
-
- if (op.getChildOperators() != null) {
- opList.addAll(op.getChildOperators());
- }
- }
- }
- }
- }
- }
-
- @Override
public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
for (Operator<? extends OperatorDesc> op : work.getAliasToWork().values()) {
op.updateCounters(ctrs);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java Tue Jul 9 09:07:35 2013
@@ -39,7 +39,6 @@ import java.util.TreeMap;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -60,6 +59,9 @@ import org.json.JSONObject;
public class ExplainTask extends Task<ExplainWork> implements Serializable {
private static final long serialVersionUID = 1L;
public static final String EXPL_COLUMN_NAME = "Explain";
+ private Set<Operator<? extends OperatorDesc>> visitedOps = new HashSet<Operator<?>>();
+ private boolean isLogical = false;
+
public ExplainTask() {
super();
}
@@ -109,7 +111,32 @@ public class ExplainTask extends Task<Ex
return outJSONObject;
}
- static public JSONObject getJSONPlan(PrintStream out, ExplainWork work)
+ public JSONObject getJSONLogicalPlan(PrintStream out, ExplainWork work) throws Exception {
+ isLogical = true;
+
+ JSONObject outJSONObject = new JSONObject();
+ boolean jsonOutput = work.isFormatted();
+ if (jsonOutput) {
+ out = null;
+ }
+
+ if (work.getParseContext() != null) {
+ JSONObject jsonPlan = outputMap(work.getParseContext().getTopOps(),
+ "LOGICAL PLAN", out, jsonOutput, work.getExtended(), 0);
+ if (out != null) {
+ out.println();
+ }
+
+ if (jsonOutput) {
+ outJSONObject.put("LOGICAL PLAN", jsonPlan);
+ }
+ } else {
+ System.err.println("No parse context!");
+ }
+ return outJSONObject;
+ }
+
+ public JSONObject getJSONPlan(PrintStream out, ExplainWork work)
throws Exception {
// If the user asked for a formatted output, dump the json output
// in the output stream
@@ -161,13 +188,20 @@ public class ExplainTask extends Task<Ex
OutputStream outS = resFile.getFileSystem(conf).create(resFile);
out = new PrintStream(outS);
- if (work.getDependency()) {
- JSONObject jsonDependencies = getJSONDependencies(work);
- out.print(jsonDependencies);
- } else {
- JSONObject jsonPlan = getJSONPlan(out, work);
+ if (work.isLogical()) {
+ JSONObject jsonLogicalPlan = getJSONLogicalPlan(out, work);
if (work.isFormatted()) {
- out.print(jsonPlan);
+ out.print(jsonLogicalPlan);
+ }
+ } else {
+ if (work.getDependency()) {
+ JSONObject jsonDependencies = getJSONDependencies(work);
+ out.print(jsonDependencies);
+ } else {
+ JSONObject jsonPlan = getJSONPlan(out, work);
+ if (work.isFormatted()) {
+ out.print(jsonPlan);
+ }
}
}
@@ -185,7 +219,7 @@ public class ExplainTask extends Task<Ex
}
}
- private static String indentString(int indent) {
+ private String indentString(int indent) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < indent; ++i) {
sb.append(" ");
@@ -194,7 +228,7 @@ public class ExplainTask extends Task<Ex
return sb.toString();
}
- private static JSONObject outputMap(Map<?, ?> mp, String header, PrintStream out,
+ private JSONObject outputMap(Map<?, ?> mp, String header, PrintStream out,
boolean extended, boolean jsonOutput, int indent) throws Exception {
boolean first_el = true;
@@ -252,7 +286,7 @@ public class ExplainTask extends Task<Ex
return jsonOutput ? json : null;
}
- private static JSONArray outputList(List<?> l, String header, PrintStream out,
+ private JSONArray outputList(List<?> l, String header, PrintStream out,
boolean extended, boolean jsonOutput, int indent) throws Exception {
boolean first_el = true;
@@ -297,7 +331,7 @@ public class ExplainTask extends Task<Ex
return jsonOutput ? outputArray : null;
}
- private static boolean isPrintable(Object val) {
+ private boolean isPrintable(Object val) {
if (val instanceof Boolean || val instanceof String
|| val instanceof Integer || val instanceof Long || val instanceof Byte
|| val instanceof Float || val instanceof Double) {
@@ -311,8 +345,13 @@ public class ExplainTask extends Task<Ex
return false;
}
- private static JSONObject outputPlan(Serializable work, PrintStream out,
- boolean extended, boolean jsonOutput, int indent) throws Exception {
+ private JSONObject outputPlan(Serializable work,
+ PrintStream out, boolean extended, boolean jsonOutput, int indent) throws Exception {
+ return outputPlan(work, out, extended, jsonOutput, indent, "");
+ }
+
+ private JSONObject outputPlan(Serializable work, PrintStream out,
+ boolean extended, boolean jsonOutput, int indent, String appendToHeader) throws Exception {
// Check if work has an explain annotation
Annotation note = work.getClass().getAnnotation(Explain.class);
@@ -324,7 +363,11 @@ public class ExplainTask extends Task<Ex
keyJSONObject = xpl_note.displayName();
if (out != null) {
out.print(indentString(indent));
- out.println(xpl_note.displayName());
+ if (appendToHeader != null && !appendToHeader.isEmpty()) {
+ out.println(xpl_note.displayName() + appendToHeader);
+ } else {
+ out.println(xpl_note.displayName());
+ }
}
}
}
@@ -336,18 +379,23 @@ public class ExplainTask extends Task<Ex
Operator<? extends OperatorDesc> operator =
(Operator<? extends OperatorDesc>) work;
if (operator.getConf() != null) {
+ String appender = isLogical ? " (" + operator.getOperatorId() + ")" : "";
JSONObject jsonOut = outputPlan(operator.getConf(), out, extended,
- jsonOutput, jsonOutput ? 0 : indent);
+ jsonOutput, jsonOutput ? 0 : indent, appender);
if (jsonOutput) {
json.put(operator.getOperatorId(), jsonOut);
}
}
- if (operator.getChildOperators() != null) {
- for (Operator<? extends OperatorDesc> op : operator.getChildOperators()) {
- JSONObject jsonOut = outputPlan(op, out, extended, jsonOutput, jsonOutput ? 0 : indent + 2);
- if (jsonOutput) {
- json.put(operator.getOperatorId(), jsonOut);
+ if (!visitedOps.contains(operator) || !isLogical) {
+ visitedOps.add(operator);
+ if (operator.getChildOperators() != null) {
+ int cindent = jsonOutput ? 0 : indent + 2;
+ for (Operator<? extends OperatorDesc> op : operator.getChildOperators()) {
+ JSONObject jsonOut = outputPlan(op, out, extended, jsonOutput, cindent);
+ if (jsonOutput) {
+ json.put(operator.getOperatorId(), jsonOut);
+ }
}
}
}
@@ -483,14 +531,14 @@ public class ExplainTask extends Task<Ex
* @param val
* @return
*/
- private static boolean shouldPrint(Explain exp, Object val) {
+ private boolean shouldPrint(Explain exp, Object val) {
if (exp.displayOnlyOnTrue() && (val instanceof Boolean) & !((Boolean)val)) {
return false;
}
return true;
}
- private static JSONObject outputPlan(Task<? extends Serializable> task,
+ private JSONObject outputPlan(Task<? extends Serializable> task,
PrintStream out, JSONObject parentJSON, boolean extended,
boolean jsonOutput, HashSet<Task<? extends Serializable>> displayedSet,
int indent) throws Exception {
@@ -534,7 +582,7 @@ public class ExplainTask extends Task<Ex
return null;
}
- private static JSONObject outputDependencies(Task<? extends Serializable> task,
+ private JSONObject outputDependencies(Task<? extends Serializable> task,
Set<Task<? extends Serializable>> dependeciesTaskSet, PrintStream out,
JSONObject parentJson, boolean jsonOutput, int indent,
boolean rootTskCandidate) throws Exception {
@@ -640,7 +688,7 @@ public class ExplainTask extends Task<Ex
return jsonOutput ? json : null;
}
- public static String outputAST(String treeString, PrintStream out,
+ public String outputAST(String treeString, PrintStream out,
boolean jsonOutput, int indent) throws JSONException {
if (out != null) {
out.print(indentString(indent));
@@ -652,7 +700,7 @@ public class ExplainTask extends Task<Ex
return jsonOutput ? treeString : null;
}
- public static JSONObject outputDependencies(PrintStream out, boolean jsonOutput,
+ public JSONObject outputDependencies(PrintStream out, boolean jsonOutput,
List<Task<? extends Serializable>> rootTasks, int indent)
throws Exception {
if (out != null) {
@@ -676,7 +724,7 @@ public class ExplainTask extends Task<Ex
return jsonOutput ? json : null;
}
- public static JSONObject outputStagePlans(PrintStream out, ExplainWork work,
+ public JSONObject outputStagePlans(PrintStream out, ExplainWork work,
List<Task<? extends Serializable>> rootTasks, int indent)
throws Exception {
boolean jsonOutput = work.isFormatted();
@@ -698,10 +746,8 @@ public class ExplainTask extends Task<Ex
* MethodComparator.
*
*/
- public static class MethodComparator implements Comparator {
- public int compare(Object o1, Object o2) {
- Method m1 = (Method) o1;
- Method m2 = (Method) o2;
+ public class MethodComparator implements Comparator<Method> {
+ public int compare(Method m1, Method m2) {
return m1.getName().compareTo(m2.getName());
}
}
@@ -717,13 +763,6 @@ public class ExplainTask extends Task<Ex
}
@Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // explain task has nothing to localize
- // we don't expect to enter this code path at all
- throw new RuntimeException("Unexpected call");
- }
-
- @Override
public List<FieldSchema> getResultSchema() {
FieldSchema tmpFieldSchema = new FieldSchema();
List<FieldSchema> colList = new ArrayList<FieldSchema>();
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java Tue Jul 9 09:07:35 2013
@@ -90,6 +90,13 @@ public abstract class ExprNodeEvaluator<
public boolean isDeterministic() {
return true;
}
+
+ /**
+ * Return whether this node (or any children nodes) are stateful.
+ */
+ public boolean isStateful() {
+ return false;
+ }
/**
* Return child evaluators if exist