You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/06/03 20:43:58 UTC
svn commit: r1489102 [2/2] - in /hive/branches/vectorization: ./
beeline/src/java/org/apache/hive/beeline/
cli/src/java/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/conf/
eclipse-templates/ hcatalog/core/src/test/...
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Mon Jun 3 18:43:56 2013
@@ -61,6 +61,7 @@ import org.apache.hadoop.hive.ql.parse.S
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
@@ -695,12 +696,14 @@ public final class ColumnPrunerProcFacto
ReduceSinkOperator reduce, ColumnPrunerProcCtx cppCtx) throws SemanticException {
ReduceSinkDesc reduceConf = reduce.getConf();
Map<String, ExprNodeDesc> oldMap = reduce.getColumnExprMap();
+ LOG.info("RS " + reduce.getIdentifier() + " oldColExprMap: " + oldMap);
RowResolver oldRR = cppCtx.getOpToParseCtxMap().get(reduce).getRowResolver();
ArrayList<ColumnInfo> signature = oldRR.getRowSchema().getSignature();
List<String> valueColNames = reduceConf.getOutputValueColumnNames();
ArrayList<String> newValueColNames = new ArrayList<String>();
+ List<ExprNodeDesc> keyExprs = reduceConf.getKeyCols();
List<ExprNodeDesc> valueExprs = reduceConf.getValueCols();
ArrayList<ExprNodeDesc> newValueExprs = new ArrayList<ExprNodeDesc>();
@@ -713,10 +716,16 @@ public final class ColumnPrunerProcFacto
outputCol = Utilities.ReduceField.VALUE.toString() + "." + outputCol;
nm = oldRR.reverseLookup(outputCol);
}
- ColumnInfo colInfo = oldRR.getFieldMap(nm[0]).remove(nm[1]);
- oldRR.getInvRslvMap().remove(colInfo.getInternalName());
- oldMap.remove(outputCol);
- signature.remove(colInfo);
+
+ // Only remove information of a column if it is not a key,
+ // i.e. this column is not appearing in keyExprs of the RS
+ if (ExprNodeDescUtils.indexOf(outputColExpr, keyExprs) == -1) {
+ ColumnInfo colInfo = oldRR.getFieldMap(nm[0]).remove(nm[1]);
+ oldRR.getInvRslvMap().remove(colInfo.getInternalName());
+ oldMap.remove(outputCol);
+ signature.remove(colInfo);
+ }
+
} else {
newValueColNames.add(outputCol);
newValueExprs.add(outputColExpr);
@@ -729,6 +738,7 @@ public final class ColumnPrunerProcFacto
.getFieldSchemasFromColumnList(reduceConf.getValueCols(),
newValueColNames, 0, ""));
reduceConf.setValueSerializeInfo(newValueTable);
+ LOG.info("RS " + reduce.getIdentifier() + " newColExprMap: " + oldMap);
}
/**
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java Mon Jun 3 18:43:56 2013
@@ -41,7 +41,7 @@ implements BigTableSelectorForAutoSMJ {
public int getBigTablePosition(ParseContext parseCtx, JoinOperator joinOp)
throws SemanticException {
int bigTablePos = 0;
- long maxSize = 0;
+ long maxSize = -1;
HiveConf conf = parseCtx.getConf();
try {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java Mon Jun 3 18:43:56 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.exec.Fi
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.MapRedTask;
import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -364,6 +365,17 @@ public class CommonJoinTaskDispatcher ex
return;
}
+ // remove the unnecessary TableScan
+ if (childAliasOp instanceof TableScanOperator) {
+ TableScanOperator tso = (TableScanOperator)childAliasOp;
+ if (tso.getNumChild() != 1) {
+ // shouldn't happen
+ return;
+ }
+ childAliasOp = tso.getChildOperators().get(0);
+ childAliasOp.getParentOperators().remove(tso);
+ }
+
// Merge the 2 trees - remove the FileSinkOperator from the first tree pass it to the
// top of the second
Operator<? extends Serializable> parentFOp = mapJoinTaskFileSinkOperator
@@ -466,7 +478,7 @@ public class CommonJoinTaskDispatcher ex
HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD);
boolean bigTableFound = false;
- long largestBigTableCandidateSize = 0;
+ long largestBigTableCandidateSize = -1;
long sumTableSizes = 0;
for (String alias : aliasToWork.keySet()) {
int tablePosition = getPosition(currWork, joinOp, alias);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Mon Jun 3 18:43:56 2013
@@ -64,7 +64,6 @@ import org.apache.hadoop.hive.ql.plan.Li
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
@@ -121,7 +120,6 @@ public abstract class BaseSemanticAnalyz
.getName();
protected static final String ORCFILE_SERDE = OrcSerde.class
.getName();
- protected static final String COLUMNAR_SERDE = ColumnarSerDe.class.getName();
class RowFormatParams {
String fieldDelim = null;
@@ -195,7 +193,7 @@ public abstract class BaseSemanticAnalyz
inputFormat = RCFILE_INPUT;
outputFormat = RCFILE_OUTPUT;
if (shared.serde == null) {
- shared.serde = COLUMNAR_SERDE;
+ shared.serde = conf.getVar(HiveConf.ConfVars.HIVEDEFAULTRCFILESERDE);
}
storageFormat = true;
break;
@@ -231,7 +229,7 @@ public abstract class BaseSemanticAnalyz
} else if ("RCFile".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT))) {
inputFormat = RCFILE_INPUT;
outputFormat = RCFILE_OUTPUT;
- shared.serde = COLUMNAR_SERDE;
+ shared.serde = conf.getVar(HiveConf.ConfVars.HIVEDEFAULTRCFILESERDE);
} else if ("ORC".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT))) {
inputFormat = ORCFILE_INPUT;
outputFormat = ORCFILE_OUTPUT;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Mon Jun 3 18:43:56 2013
@@ -1350,7 +1350,7 @@ public class DDLSemanticAnalyzer extends
case HiveParser.TOK_TBLRCFILE:
inputFormat = RCFILE_INPUT;
outputFormat = RCFILE_OUTPUT;
- serde = COLUMNAR_SERDE;
+ serde = conf.getVar(HiveConf.ConfVars.HIVEDEFAULTRCFILESERDE);
break;
case HiveParser.TOK_TBLORCFILE:
inputFormat = ORCFILE_INPUT;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java Mon Jun 3 18:43:56 2013
@@ -95,6 +95,7 @@ public final class SemanticAnalyzerFacto
commandType.put(HiveParser.TOK_DESCDATABASE, HiveOperation.DESCDATABASE);
commandType.put(HiveParser.TOK_ALTERTABLE_SKEWED, HiveOperation.ALTERTABLE_SKEWED);
commandType.put(HiveParser.TOK_ANALYZE, HiveOperation.ANALYZE_TABLE);
+ commandType.put(HiveParser.TOK_ALTERVIEW_RENAME, HiveOperation.ALTERVIEW_RENAME);
}
static {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java Mon Jun 3 18:43:56 2013
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.plan;
import org.apache.hadoop.hive.ql.security.authorization.Privilege;
public enum HiveOperation {
-
EXPLAIN("EXPLAIN", null, null),
LOAD("LOAD", null, new Privilege[]{Privilege.ALTER_DATA}),
EXPORT("EXPORT", new Privilege[]{Privilege.SELECT}, null),
@@ -102,6 +101,7 @@ public enum HiveOperation {
ALTERTABLE_SKEWED("ALTERTABLE_SKEWED", new Privilege[] {Privilege.ALTER_METADATA}, null),
ALTERTBLPART_SKEWED_LOCATION("ALTERTBLPART_SKEWED_LOCATION",
new Privilege[] {Privilege.ALTER_DATA}, null),
+ ALTERVIEW_RENAME("ALTERVIEW_RENAME", new Privilege[] {Privilege.ALTER_METADATA}, null),
;
private String operationName;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Mon Jun 3 18:43:56 2013
@@ -22,16 +22,18 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
+import java.lang.management.ManagementFactory;
import java.net.URI;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.GregorianCalendar;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -57,6 +59,7 @@ import org.apache.hadoop.hive.ql.util.Do
* configuration information
*/
public class SessionState {
+ private static final Log LOG = LogFactory.getLog(SessionState.class);
/**
* current configuration.
@@ -179,18 +182,21 @@ public class SessionState {
this.isVerbose = isVerbose;
}
- public SessionState() {
- this(null);
- }
-
public SessionState(HiveConf conf) {
this.conf = conf;
isSilent = conf.getBoolVar(HiveConf.ConfVars.HIVESESSIONSILENT);
ls = new LineageState();
overriddenConfigurations = new HashMap<String, String>();
overriddenConfigurations.putAll(HiveConf.getConfSystemProperties());
+ // if there isn't already a session name, go ahead and create it.
+ if (StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HIVESESSIONID))) {
+ conf.setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId());
+ }
}
+ private static final SimpleDateFormat DATE_FORMAT =
+ new SimpleDateFormat("yyyyMMddHHmm");
+
public void setCmd(String cmdString) {
conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, cmdString);
}
@@ -242,12 +248,6 @@ public class SessionState {
tss.set(startSs);
- if (StringUtils.isEmpty(startSs.getConf().getVar(
- HiveConf.ConfVars.HIVESESSIONID))) {
- startSs.getConf()
- .setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId());
- }
-
if (startSs.hiveHist == null) {
startSs.hiveHist = new HiveHistory(startSs);
}
@@ -297,15 +297,15 @@ public class SessionState {
return hiveHist;
}
+ /**
+ * Create a session ID. Looks like:
+ * $user_$pid@$host_$date
+ * @return the unique string
+ */
private static String makeSessionId() {
- GregorianCalendar gc = new GregorianCalendar();
String userid = System.getProperty("user.name");
-
- return userid
- + "_"
- + String.format("%1$4d%2$02d%3$02d%4$02d%5$02d", gc.get(Calendar.YEAR),
- gc.get(Calendar.MONTH) + 1, gc.get(Calendar.DAY_OF_MONTH), gc
- .get(Calendar.HOUR_OF_DAY), gc.get(Calendar.MINUTE));
+ return userid + "_" + ManagementFactory.getRuntimeMXBean().getName() + "_"
+ + DATE_FORMAT.format(new Date());
}
/**
@@ -588,35 +588,15 @@ public class SessionState {
private String downloadResource(String value, boolean convertToUnix) {
if (canDownloadResource(value)) {
getConsole().printInfo("converting to local " + value);
- String location = getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR);
-
+ File resourceDir = new File(getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
String destinationName = new Path(value).getName();
- String prefix = destinationName;
- String postfix = null;
- int index = destinationName.lastIndexOf(".");
- if (index > 0) {
- prefix = destinationName.substring(0, index);
- postfix = destinationName.substring(index);
- }
- if (prefix.length() < 3) {
- prefix += ".tmp"; // prefix should be longer than 3
- }
-
- File resourceDir = new File(location);
- if (resourceDir.exists() && !resourceDir.isDirectory()) {
- throw new RuntimeException("The resource directory is not a directory, " +
- "resourceDir is set to " + resourceDir);
+ File destinationFile = new File(resourceDir, destinationName);
+ if (resourceDir.exists() && ! resourceDir.isDirectory()) {
+ throw new RuntimeException("The resource directory is not a directory, resourceDir is set to" + resourceDir);
}
if (!resourceDir.exists() && !resourceDir.mkdirs()) {
throw new RuntimeException("Couldn't create directory " + resourceDir);
}
-
- File destinationFile;
- try {
- destinationFile = File.createTempFile(prefix, postfix, resourceDir);
- } catch (Exception e) {
- throw new RuntimeException("Failed to create temporary file for " + value, e);
- }
try {
FileSystem fs = FileSystem.get(new URI(value), conf);
fs.copyToLocalFile(new Path(value), new Path(destinationFile.getCanonicalPath()));
@@ -756,4 +736,17 @@ public class SessionState {
public void setLocalMapRedErrors(Map<String, List<String>> localMapRedErrors) {
this.localMapRedErrors = localMapRedErrors;
}
+
+ public void close() throws IOException {
+ File resourceDir =
+ new File(getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+ LOG.debug("Removing resource dir " + resourceDir);
+ try {
+ if (resourceDir.exists()) {
+ FileUtils.deleteDirectory(resourceDir);
+ }
+ } catch (IOException e) {
+ LOG.info("Error removing session resource dir " + resourceDir, e);
+ }
+ }
}
Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Mon Jun 3 18:43:56 2013
@@ -1023,7 +1023,8 @@ public class QTestUtil {
".*LOCK_TIME:.*",
".*grantTime.*",
".*[.][.][.] [0-9]* more.*",
- ".*job_[0-9]*_[0-9]*.*",
+ ".*job_[0-9_]*.*",
+ ".*job_local[0-9_]*.*",
".*USING 'java -cp.*",
"^Deleted.*",
};
Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/join_vc.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/join_vc.q?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/join_vc.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/join_vc.q Mon Jun 3 18:43:56 2013
@@ -1,5 +1,5 @@
-- see HIVE-4033 earlier a flag named hasVC was not initialized correctly in MapOperator.java, resulting in NPE for following query. order by and limit in the query is not relevant, problem would be evident even without those. They are there to keep .q.out file small and sorted.
-explain select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.value limit 3;
+explain select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value limit 3;
-select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.value limit 3;
+select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value limit 3;
Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/rcfile_default_format.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/rcfile_default_format.q?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/rcfile_default_format.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/rcfile_default_format.q Mon Jun 3 18:43:56 2013
@@ -1,19 +1,31 @@
SET hive.default.fileformat = RCFile;
CREATE TABLE rcfile_default_format (key STRING);
-DESCRIBE EXTENDED rcfile_default_format;
+DESCRIBE FORMATTED rcfile_default_format;
CREATE TABLE rcfile_default_format_ctas AS SELECT key,value FROM src;
-DESCRIBE EXTENDED rcfile_default_format_ctas;
+DESCRIBE FORMATTED rcfile_default_format_ctas;
CREATE TABLE rcfile_default_format_txtfile (key STRING) STORED AS TEXTFILE;
INSERT OVERWRITE TABLE rcfile_default_format_txtfile SELECT key from src;
-DESCRIBE EXTENDED rcfile_default_format_txtfile;
+DESCRIBE FORMATTED rcfile_default_format_txtfile;
SET hive.default.fileformat = TextFile;
CREATE TABLE textfile_default_format_ctas AS SELECT key,value FROM rcfile_default_format_ctas;
-DESCRIBE EXTENDED textfile_default_format_ctas;
+DESCRIBE FORMATTED textfile_default_format_ctas;
+SET hive.default.fileformat = RCFile;
+SET hive.default.rcfile.serde = org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe;
+CREATE TABLE rcfile_default_format_ctas_default_serde AS SELECT key,value FROM rcfile_default_format_ctas;
+DESCRIBE FORMATTED rcfile_default_format_ctas_default_serde;
+
+CREATE TABLE rcfile_default_format_default_serde (key STRING);
+DESCRIBE FORMATTED rcfile_default_format_default_serde;
+SET hive.default.fileformat = TextFile;
+CREATE TABLE rcfile_ctas_default_serde STORED AS rcfile AS SELECT key,value FROM rcfile_default_format_ctas;
+DESCRIBE FORMATTED rcfile_ctas_default_serde;
+CREATE TABLE rcfile_default_serde (key STRING) STORED AS rcfile;
+DESCRIBE FORMATTED rcfile_default_serde;
Modified: hive/branches/vectorization/ql/src/test/results/clientnegative/local_mapred_error_cache.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientnegative/local_mapred_error_cache.q.out?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientnegative/local_mapred_error_cache.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientnegative/local_mapred_error_cache.q.out Mon Jun 3 18:43:56 2013
@@ -19,6 +19,5 @@ org.apache.hadoop.hive.ql.metadata.HiveE
#### A masked pattern was here ####
org.apache.hadoop.hive.ql.metadata.HiveException: [Error 20003]: An error occurred when trying to close the Operator running your custom script.
#### A masked pattern was here ####
-Ended Job = job_local_0001 with errors
Error during job, obtaining debugging information...
FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.MapRedTask
Modified: hive/branches/vectorization/ql/src/test/results/clientnegative/recursive_view.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientnegative/recursive_view.q.out?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientnegative/recursive_view.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientnegative/recursive_view.q.out Mon Jun 3 18:43:56 2013
@@ -68,11 +68,11 @@ POSTHOOK: type: DROPVIEW
POSTHOOK: Input: default@r0
POSTHOOK: Output: default@r0
PREHOOK: query: alter view r3 rename to r0
-PREHOOK: type: null
+PREHOOK: type: ALTERVIEW_RENAME
PREHOOK: Input: default@r3
PREHOOK: Output: default@r3
POSTHOOK: query: alter view r3 rename to r0
-POSTHOOK: type: null
+POSTHOOK: type: ALTERVIEW_RENAME
POSTHOOK: Input: default@r3
POSTHOOK: Output: default@r0
POSTHOOK: Output: default@r3
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/alter_view_rename.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/alter_view_rename.q.out?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/alter_view_rename.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/alter_view_rename.q.out Mon Jun 3 18:43:56 2013
@@ -18,11 +18,11 @@ ds string
#### A masked pattern was here ####
PREHOOK: query: ALTER VIEW view1 RENAME TO view2
-PREHOOK: type: null
+PREHOOK: type: ALTERVIEW_RENAME
PREHOOK: Input: default@view1
PREHOOK: Output: default@view1
POSTHOOK: query: ALTER VIEW view1 RENAME TO view2
-POSTHOOK: type: null
+POSTHOOK: type: ALTERVIEW_RENAME
POSTHOOK: Input: default@view1
POSTHOOK: Output: default@view1
POSTHOOK: Output: default@view2
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/join_vc.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/join_vc.q.out?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/join_vc.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/join_vc.q.out Mon Jun 3 18:43:56 2013
@@ -1,13 +1,13 @@
PREHOOK: query: -- see HIVE-4033 earlier a flag named hasVC was not initialized correctly in MapOperator.java, resulting in NPE for following query. order by and limit in the query is not relevant, problem would be evident even without those. They are there to keep .q.out file small and sorted.
-explain select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.value limit 3
+explain select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value limit 3
PREHOOK: type: QUERY
POSTHOOK: query: -- see HIVE-4033 earlier a flag named hasVC was not initialized correctly in MapOperator.java, resulting in NPE for following query. order by and limit in the query is not relevant, problem would be evident even without those. They are there to keep .q.out file small and sorted.
-explain select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.value limit 3
+explain select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value limit 3
POSTHOOK: type: QUERY
ABSTRACT SYNTAX TREE:
- (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) t1) (TOK_TABREF (TOK_TABNAME src) t2) (= (. (TOK_TABLE_OR_COL t1) key) (. (TOK_TABLE_OR_COL t2) key))) (TOK_TABREF (TOK_TABNAME src) t3) (= (. (TOK_TABLE_OR_COL t2) value) (. (TOK_TABLE_OR_COL t3) value)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL t3) BLOCK__OFFSET__INSIDE__FILE)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL t3) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL t3) value))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL t3) value))) (TOK_LIMIT 3)))
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) t1) (TOK_TABREF (TOK_TABNAME src) t2) (= (. (TOK_TABLE_OR_COL t1) key) (. (TOK_TABLE_OR_COL t2) key))) (TOK_TABREF (TOK_TABNAME src) t3) (= (. (TOK_TABLE_OR_COL t2) value) (. (TOK_TABLE_OR_COL t3) value)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL t3) BLOCK__OFFSET__INSIDE__FILE)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL t3) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL t3) value))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL t3) BLOCK__OFFSET__INSIDE__FILE)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL t3) key)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL t3) value))) (TOK_LIMIT 3)))
STAGE DEPENDENCIES:
Stage-3 is a root stage
@@ -125,9 +125,13 @@ STAGE PLANS:
#### A masked pattern was here ####
Reduce Output Operator
key expressions:
+ expr: _col0
+ type: bigint
+ expr: _col1
+ type: string
expr: _col2
type: string
- sort order: +
+ sort order: +++
tag: -1
value expressions:
expr: _col0
@@ -151,14 +155,14 @@ STAGE PLANS:
limit: 3
-PREHOOK: query: select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.value limit 3
+PREHOOK: query: select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value limit 3
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.value limit 3
+POSTHOOK: query: select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value limit 3
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-2088 0 val_0
-2632 0 val_0
-968 0 val_0
+0 238 val_238
+0 238 val_238
+0 238 val_238
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/multiMapJoin1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/multiMapJoin1.q.out?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/multiMapJoin1.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/multiMapJoin1.q.out Mon Jun 3 18:43:56 2013
@@ -598,18 +598,18 @@ STAGE PLANS:
type: string
mode: hash
outputColumnNames: _col0, _col1
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: _col0
- type: string
- tag: -1
- value expressions:
- expr: _col1
- type: bigint
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -1836,12 +1836,12 @@ STAGE PLANS:
bucketGroup: false
mode: hash
outputColumnNames: _col0
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/rcfile_default_format.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/rcfile_default_format.q.out?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/rcfile_default_format.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/rcfile_default_format.q.out Mon Jun 3 18:43:56 2013
@@ -3,13 +3,34 @@ PREHOOK: type: CREATETABLE
POSTHOOK: query: CREATE TABLE rcfile_default_format (key STRING)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@rcfile_default_format
-PREHOOK: query: DESCRIBE EXTENDED rcfile_default_format
+PREHOOK: query: DESCRIBE FORMATTED rcfile_default_format
PREHOOK: type: DESCTABLE
-POSTHOOK: query: DESCRIBE EXTENDED rcfile_default_format
+POSTHOOK: query: DESCRIBE FORMATTED rcfile_default_format
POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
key string None
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
PREHOOK: query: CREATE TABLE rcfile_default_format_ctas AS SELECT key,value FROM src
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@src
@@ -17,14 +38,40 @@ POSTHOOK: query: CREATE TABLE rcfile_def
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@src
POSTHOOK: Output: default@rcfile_default_format_ctas
-PREHOOK: query: DESCRIBE EXTENDED rcfile_default_format_ctas
+PREHOOK: query: DESCRIBE FORMATTED rcfile_default_format_ctas
PREHOOK: type: DESCTABLE
-POSTHOOK: query: DESCRIBE EXTENDED rcfile_default_format_ctas
+POSTHOOK: query: DESCRIBE FORMATTED rcfile_default_format_ctas
POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
key string None
value string None
+# Detailed Table Information
+Database: default
#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ numFiles 1
+ numPartitions 0
+ numRows 500
+ rawDataSize 4812
+ totalSize 5293
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
PREHOOK: query: CREATE TABLE rcfile_default_format_txtfile (key STRING) STORED AS TEXTFILE
PREHOOK: type: CREATETABLE
POSTHOOK: query: CREATE TABLE rcfile_default_format_txtfile (key STRING) STORED AS TEXTFILE
@@ -39,14 +86,40 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@rcfile_default_format_txtfile
POSTHOOK: Lineage: rcfile_default_format_txtfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: DESCRIBE EXTENDED rcfile_default_format_txtfile
+PREHOOK: query: DESCRIBE FORMATTED rcfile_default_format_txtfile
PREHOOK: type: DESCTABLE
-POSTHOOK: query: DESCRIBE EXTENDED rcfile_default_format_txtfile
+POSTHOOK: query: DESCRIBE FORMATTED rcfile_default_format_txtfile
POSTHOOK: type: DESCTABLE
POSTHOOK: Lineage: rcfile_default_format_txtfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+# col_name data_type comment
+
key string None
+# Detailed Table Information
+Database: default
#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ numFiles 1
+ numPartitions 0
+ numRows 500
+ rawDataSize 1406
+ totalSize 1906
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
PREHOOK: query: CREATE TABLE textfile_default_format_ctas AS SELECT key,value FROM rcfile_default_format_ctas
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@rcfile_default_format_ctas
@@ -55,12 +128,194 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@rcfile_default_format_ctas
POSTHOOK: Output: default@textfile_default_format_ctas
POSTHOOK: Lineage: rcfile_default_format_txtfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: DESCRIBE EXTENDED textfile_default_format_ctas
+PREHOOK: query: DESCRIBE FORMATTED textfile_default_format_ctas
PREHOOK: type: DESCTABLE
-POSTHOOK: query: DESCRIBE EXTENDED textfile_default_format_ctas
+POSTHOOK: query: DESCRIBE FORMATTED textfile_default_format_ctas
POSTHOOK: type: DESCTABLE
POSTHOOK: Lineage: rcfile_default_format_txtfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+# col_name data_type comment
+
key string None
value string None
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ numFiles 1
+ numPartitions 0
+ numRows 500
+ rawDataSize 5312
+ totalSize 5812
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: CREATE TABLE rcfile_default_format_ctas_default_serde AS SELECT key,value FROM rcfile_default_format_ctas
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@rcfile_default_format_ctas
+POSTHOOK: query: CREATE TABLE rcfile_default_format_ctas_default_serde AS SELECT key,value FROM rcfile_default_format_ctas
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@rcfile_default_format_ctas
+POSTHOOK: Output: default@rcfile_default_format_ctas_default_serde
+POSTHOOK: Lineage: rcfile_default_format_txtfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE FORMATTED rcfile_default_format_ctas_default_serde
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE FORMATTED rcfile_default_format_ctas_default_serde
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Lineage: rcfile_default_format_txtfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+# col_name data_type comment
+
+key string from deserializer
+value string from deserializer
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ numFiles 1
+ numPartitions 0
+ numRows 500
+ rawDataSize 4812
+ totalSize 5293
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: CREATE TABLE rcfile_default_format_default_serde (key STRING)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE rcfile_default_format_default_serde (key STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@rcfile_default_format_default_serde
+POSTHOOK: Lineage: rcfile_default_format_txtfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE FORMATTED rcfile_default_format_default_serde
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE FORMATTED rcfile_default_format_default_serde
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Lineage: rcfile_default_format_txtfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+# col_name data_type comment
+
+key string from deserializer
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: CREATE TABLE rcfile_ctas_default_serde STORED AS rcfile AS SELECT key,value FROM rcfile_default_format_ctas
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@rcfile_default_format_ctas
+POSTHOOK: query: CREATE TABLE rcfile_ctas_default_serde STORED AS rcfile AS SELECT key,value FROM rcfile_default_format_ctas
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@rcfile_default_format_ctas
+POSTHOOK: Output: default@rcfile_ctas_default_serde
+POSTHOOK: Lineage: rcfile_default_format_txtfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE FORMATTED rcfile_ctas_default_serde
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE FORMATTED rcfile_ctas_default_serde
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Lineage: rcfile_default_format_txtfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+# col_name data_type comment
+
+key string from deserializer
+value string from deserializer
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ numFiles 1
+ numPartitions 0
+ numRows 500
+ rawDataSize 4812
+ totalSize 5293
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: CREATE TABLE rcfile_default_serde (key STRING) STORED AS rcfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE rcfile_default_serde (key STRING) STORED AS rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@rcfile_default_serde
+POSTHOOK: Lineage: rcfile_default_format_txtfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: DESCRIBE FORMATTED rcfile_default_serde
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE FORMATTED rcfile_default_serde
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Lineage: rcfile_default_format_txtfile.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+# col_name data_type comment
+
+key string from deserializer
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/join1.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/join1.q.xml?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/join1.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/join1.q.xml Mon Jun 3 18:43:56 2013
@@ -486,6 +486,20 @@
</void>
</object>
</void>
+ <void method="put">
+ <string>VALUE._col0</string>
+ <object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
+ <void property="column">
+ <string>key</string>
+ </void>
+ <void property="tabAlias">
+ <string>src2</string>
+ </void>
+ <void property="typeInfo">
+ <object idref="PrimitiveTypeInfo0"/>
+ </void>
+ </object>
+ </void>
</object>
</void>
<void property="conf">
@@ -639,6 +653,19 @@
<void method="add">
<object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
<void property="internalName">
+ <string>VALUE._col0</string>
+ </void>
+ <void property="tabAlias">
+ <string>src2</string>
+ </void>
+ <void property="type">
+ <object idref="PrimitiveTypeInfo0"/>
+ </void>
+ </object>
+ </void>
+ <void method="add">
+ <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
+ <void property="internalName">
<string>VALUE._col1</string>
</void>
<void property="tabAlias">
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/join3.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/join3.q.xml?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/join3.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/join3.q.xml Mon Jun 3 18:43:56 2013
@@ -615,7 +615,26 @@
<void method="add">
<object id="ReduceSinkOperator0" class="org.apache.hadoop.hive.ql.exec.ReduceSinkOperator">
<void property="columnExprMap">
- <object class="java.util.HashMap"/>
+ <object class="java.util.HashMap">
+ <void method="put">
+ <string>VALUE._col0</string>
+ <object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
+ <void property="column">
+ <string>key</string>
+ </void>
+ <void property="tabAlias">
+ <string>src2</string>
+ </void>
+ <void property="typeInfo">
+ <object id="PrimitiveTypeInfo0" class="org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo">
+ <void property="typeName">
+ <string>string</string>
+ </void>
+ </object>
+ </void>
+ </object>
+ </void>
+ </object>
</void>
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.ReduceSinkDesc">
@@ -633,11 +652,7 @@
<string>src2</string>
</void>
<void property="typeInfo">
- <object id="PrimitiveTypeInfo0" class="org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo">
- <void property="typeName">
- <string>string</string>
- </void>
- </object>
+ <object idref="PrimitiveTypeInfo0"/>
</void>
</object>
</void>
@@ -760,7 +775,21 @@
<void property="schema">
<object class="org.apache.hadoop.hive.ql.exec.RowSchema">
<void property="signature">
- <object class="java.util.ArrayList"/>
+ <object class="java.util.ArrayList">
+ <void method="add">
+ <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
+ <void property="internalName">
+ <string>VALUE._col0</string>
+ </void>
+ <void property="tabAlias">
+ <string>src2</string>
+ </void>
+ <void property="type">
+ <object idref="PrimitiveTypeInfo0"/>
+ </void>
+ </object>
+ </void>
+ </object>
</void>
</object>
</void>
@@ -899,6 +928,20 @@
</void>
</object>
</void>
+ <void method="put">
+ <string>VALUE._col0</string>
+ <object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
+ <void property="column">
+ <string>key</string>
+ </void>
+ <void property="tabAlias">
+ <string>src3</string>
+ </void>
+ <void property="typeInfo">
+ <object idref="PrimitiveTypeInfo0"/>
+ </void>
+ </object>
+ </void>
</object>
</void>
<void property="conf">
@@ -1052,6 +1095,19 @@
<void method="add">
<object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
<void property="internalName">
+ <string>VALUE._col0</string>
+ </void>
+ <void property="tabAlias">
+ <string>src3</string>
+ </void>
+ <void property="type">
+ <object idref="PrimitiveTypeInfo0"/>
+ </void>
+ </object>
+ </void>
+ <void method="add">
+ <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
+ <void property="internalName">
<string>VALUE._col1</string>
</void>
<void property="tabAlias">
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/TUGIContainingProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/TUGIContainingProcessor.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/TUGIContainingProcessor.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/TUGIContainingProcessor.java Mon Jun 3 18:43:56 2013
@@ -54,12 +54,5 @@ public class TUGIContainingProcessor imp
} catch (IOException ioe) {
throw new RuntimeException(ioe); // unexpected!
}
- finally {
- // cleanup the filesystem handles at the end if they are cached
- // clientUgi will be null if createRemoteUser() fails
- if (clientUgi != null && !isFsCacheDisabled) {
- shim.closeAllForUGI(clientUgi);
- }
- }
}
}
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java Mon Jun 3 18:43:56 2013
@@ -79,6 +79,7 @@ public abstract class HiveCommandOperati
sessionState.out = new PrintStream(System.out, true, "UTF-8");
sessionState.err = new PrintStream(System.err, true, "UTF-8");
} catch (UnsupportedEncodingException ee) {
+ LOG.error("Error creating PrintStream", e);
ee.printStackTrace();
sessionState.out = null;
sessionState.err = null;
@@ -117,7 +118,7 @@ public abstract class HiveCommandOperati
}
} catch (Exception e) {
setState(OperationState.ERROR);
- throw new HiveSQLException("Error running query: " + e.toString());
+ throw new HiveSQLException("Error running query: " + e.toString(), e);
}
setState(OperationState.FINISHED);
}
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java Mon Jun 3 18:43:56 2013
@@ -110,7 +110,7 @@ public class SQLOperation extends Execut
throw e;
} catch (Exception e) {
setState(OperationState.ERROR);
- throw new HiveSQLException("Error running query: " + e.toString());
+ throw new HiveSQLException("Error running query: " + e.toString(), e);
}
setState(OperationState.FINISHED);
}
@@ -208,7 +208,7 @@ public class SQLOperation extends Execut
// for now, expose non-primitive as a string
// TODO: expose non-primitive as a structured object while maintaining JDBC compliance
if (oi.getCategory() != ObjectInspector.Category.PRIMITIVE) {
- return SerDeUtils.getJSONString(o, oi);
+ return SerDeUtils.getJSONString(o, oi);
}
return obj;
}
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1489102&r1=1489101&r2=1489102&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java Mon Jun 3 18:43:56 2013
@@ -18,13 +18,19 @@
package org.apache.hive.service.cli.session;
+import java.io.File;
+import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -64,6 +70,8 @@ public class HiveSessionImpl implements
private static final String FETCH_WORK_SERDE_CLASS =
"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
+ private static final Log LOG = LogFactory.getLog(HiveSessionImpl.class);
+
private SessionManager sessionManager;
private OperationManager operationManager;
@@ -79,7 +87,9 @@ public class HiveSessionImpl implements
hiveConf.set(entry.getKey(), entry.getValue());
}
}
-
+ // set an explicit session name to control the download directory name
+ hiveConf.set(ConfVars.HIVESESSIONID.varname,
+ sessionHandle.getHandleIdentifier().toString());
sessionState = new SessionState(hiveConf);
}
@@ -300,8 +310,11 @@ public class HiveSessionImpl implements
if (null != hiveHist) {
hiveHist.closeStream();
}
- } finally {
+ sessionState.close();
+ release();
+ } catch (IOException ioe) {
release();
+ throw new HiveSQLException("Failure to close", ioe);
}
}