You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/09/30 23:58:32 UTC
svn commit: r1527793 [2/3] - in /hive/branches/vectorization: ./
common/src/java/org/apache/hadoop/hive/conf/
contrib/src/java/org/apache/hadoop/hive/contrib/fileformat/base64/
data/files/ hbase-handler/src/java/org/apache/hadoop/hive/hbase/
hcatalog/c...
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughRecordWriter.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughRecordWriter.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughRecordWriter.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/HivePassThroughRecordWriter.java Mon Sep 30 21:58:29 2013
@@ -20,13 +20,12 @@ package org.apache.hadoop.hive.ql.io;
import java.io.IOException;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
public class HivePassThroughRecordWriter <K extends WritableComparable<?>, V extends Writable>
-implements RecordWriter {
+implements FSRecordWriter {
private final org.apache.hadoop.mapred.RecordWriter<K, V> mWriter;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java Mon Sep 30 21:58:29 2013
@@ -23,7 +23,6 @@ import java.util.Properties;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.SequenceFile;
@@ -56,7 +55,7 @@ public class HiveSequenceFileOutputForma
* @return the RecordWriter for the output file
*/
@Override
- public RecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath,
+ public FSRecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath,
Class<? extends Writable> valueClass, boolean isCompressed,
Properties tableProperties, Progressable progress) throws IOException {
@@ -64,7 +63,7 @@ public class HiveSequenceFileOutputForma
final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
fs, finalOutPath, BytesWritable.class, valueClass, isCompressed);
- return new RecordWriter() {
+ return new FSRecordWriter() {
public void write(Writable r) throws IOException {
outStream.append(EMPTY_KEY, r);
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java Mon Sep 30 21:58:29 2013
@@ -118,7 +118,7 @@ public class RCFileOutputFormat extends
* @throws IOException
*/
@Override
- public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter(
+ public FSRecordWriter getHiveRecordWriter(
JobConf jc, Path finalOutPath, Class<? extends Writable> valueClass,
boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException {
@@ -135,7 +135,7 @@ public class RCFileOutputFormat extends
(jc, finalOutPath.getFileSystem(jc),
finalOutPath, isCompressed);
- return new org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter() {
+ return new FSRecordWriter() {
public void write(Writable r) throws IOException {
outWriter.append(r);
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.java Mon Sep 30 21:58:29 2013
@@ -17,6 +17,14 @@
*/
package org.apache.hadoop.hive.ql.io.avro;
+import static org.apache.avro.file.DataFileConstants.DEFLATE_CODEC;
+import static org.apache.avro.mapred.AvroJob.OUTPUT_CODEC;
+import static org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL;
+import static org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY;
+
+import java.io.IOException;
+import java.util.Properties;
+
import org.apache.avro.Schema;
import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.DataFileWriter;
@@ -24,7 +32,7 @@ import org.apache.avro.generic.GenericDa
import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.io.FSRecordWriter;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable;
import org.apache.hadoop.hive.serde2.avro.AvroSerdeException;
@@ -36,14 +44,6 @@ import org.apache.hadoop.mapred.RecordWr
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.Progressable;
-import java.io.IOException;
-import java.util.Properties;
-
-import static org.apache.avro.file.DataFileConstants.DEFLATE_CODEC;
-import static org.apache.avro.mapred.AvroJob.OUTPUT_CODEC;
-import static org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL;
-import static org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY;
-
/**
* Write to an Avro file from a Hive process.
*/
@@ -51,7 +51,7 @@ public class AvroContainerOutputFormat
implements HiveOutputFormat<LongWritable, AvroGenericRecordWritable> {
@Override
- public FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jobConf,
+ public FSRecordWriter getHiveRecordWriter(JobConf jobConf,
Path path, Class<? extends Writable> valueClass, boolean isCompressed,
Properties properties, Progressable progressable) throws IOException {
Schema schema;
@@ -62,7 +62,7 @@ public class AvroContainerOutputFormat
}
GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schema);
DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw);
-
+
if (isCompressed) {
int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordWriter.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordWriter.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordWriter.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordWriter.java Mon Sep 30 21:58:29 2013
@@ -18,18 +18,18 @@
package org.apache.hadoop.hive.ql.io.avro;
+import java.io.IOException;
+
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.io.FSRecordWriter;
import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable;
import org.apache.hadoop.io.Writable;
-import java.io.IOException;
-
/**
* Write an Avro GenericRecord to an Avro data file.
*/
-public class AvroGenericRecordWriter implements FileSinkOperator.RecordWriter{
+public class AvroGenericRecordWriter implements FSRecordWriter{
final private DataFileWriter<GenericRecord> dfw;
public AvroGenericRecordWriter(DataFileWriter<GenericRecord> dfw) {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java Mon Sep 30 21:58:29 2013
@@ -17,10 +17,9 @@
*/
package org.apache.hadoop.hive.ql.io.orc;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.io.FSRecordWriter;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcSerde.OrcSerdeRow;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -45,7 +44,7 @@ public class OrcOutputFormat extends Fil
private static class OrcRecordWriter
implements RecordWriter<NullWritable, OrcSerdeRow>,
- FileSinkOperator.RecordWriter {
+ FSRecordWriter {
private Writer writer = null;
private final Path path;
private final OrcFile.WriterOptions options;
@@ -105,7 +104,7 @@ public class OrcOutputFormat extends Fil
}
@Override
- public FileSinkOperator.RecordWriter
+ public FSRecordWriter
getHiveRecordWriter(JobConf conf,
Path path,
Class<? extends Writable> valueClass,
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java Mon Sep 30 21:58:29 2013
@@ -39,6 +39,19 @@ public interface Reader {
long getNumberOfRows();
/**
+ * Get the deserialized data size of the file
+ * @return raw data size
+ */
+ long getRawDataSize();
+
+ /**
+ * Get the deserialized data size of the specified columns
+ * @param colNames
+ * @return raw data size of columns
+ */
+ long getRawDataSizeOfColumns(List<String> colNames);
+
+ /**
* Get the user metadata keys.
* @return the set of metadata keys
*/
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java Mon Sep 30 21:58:29 2013
@@ -343,4 +343,14 @@ final class ReaderImpl implements Reader
include, footer.getRowIndexStride(), sarg, columnNames);
}
+ @Override
+ public long getRawDataSize() {
+ return 0;
+ }
+
+ @Override
+ public long getRawDataSizeOfColumns(List<String> colNames) {
+ return 0;
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java Mon Sep 30 21:58:29 2013
@@ -47,4 +47,22 @@ public interface Writer {
* @throws IOException
*/
void close() throws IOException;
+
+ /**
+ * Return the deserialized data size. Raw data size will be compute when
+ * writing the file footer. Hence raw data size value will be available only
+ * after closing the writer.
+ *
+ * @return raw data size
+ */
+ long getRawDataSize();
+
+ /**
+ * Return the number of rows in file. Row count gets updated when flushing
+ * the stripes. To get accurate row count this method should be called after
+ * closing the writer.
+ *
+ * @return row count
+ */
+ long getNumberOfRows();
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java Mon Sep 30 21:58:29 2013
@@ -1871,4 +1871,14 @@ class WriterImpl implements Writer, Memo
rawWriter.close();
}
}
+
+ @Override
+ public long getRawDataSize() {
+ return 0;
+ }
+
+ @Override
+ public long getNumberOfRows() {
+ return 0;
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java Mon Sep 30 21:58:29 2013
@@ -378,7 +378,8 @@ public final class CorrelationUtilities
// copies desc of cGBYm to cGBYr and remove cGBYm and cRS
GroupByOperator cGBYm = (GroupByOperator) parent;
- cGBYr.getConf().setKeys(cGBYm.getConf().getKeys());
+ cGBYr.getConf().setKeys(ExprNodeDescUtils.backtrack(ExprNodeDescUtils.backtrack(cGBYr
+ .getConf().getKeys(), cGBYr, cRS), cRS, cGBYm));
cGBYr.getConf().setAggregators(cGBYm.getConf().getAggregators());
for (AggregationDesc aggr : cGBYm.getConf().getAggregators()) {
aggr.setMode(GenericUDAFEvaluator.Mode.COMPLETE);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Mon Sep 30 21:58:29 2013
@@ -29,6 +29,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.Tree;
@@ -40,6 +41,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryProperties;
+import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -59,10 +61,15 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
@@ -304,6 +311,13 @@ public abstract class BaseSemanticAnalyz
rootTasks = new ArrayList<Task<? extends Serializable>>();
}
+ public static String stripIdentifierQuotes(String val) {
+ if ((val.charAt(0) == '`' && val.charAt(val.length() - 1) == '`')) {
+ val = val.substring(1, val.length() - 1);
+ }
+ return val;
+ }
+
public static String stripQuotes(String val) {
return PlanUtils.stripQuotes(val);
}
@@ -580,7 +594,7 @@ public abstract class BaseSemanticAnalyz
// child 2 is the optional comment of the column
if (child.getChildCount() == 3) {
col.setComment(unescapeSQLString(child.getChild(2).getText()));
- }
+ }
}
colList.add(col);
}
@@ -748,7 +762,7 @@ public abstract class BaseSemanticAnalyz
}
// check if the columns specified in the partition() clause are actually partition columns
- Utilities.validatePartSpec(tableHandle, partSpec);
+ validatePartSpec(tableHandle, partSpec, ast, conf);
// check if the partition spec is valid
if (numDynParts > 0) {
@@ -1115,4 +1129,79 @@ public abstract class BaseSemanticAnalyz
return storedAsDirs;
}
+ private static void getPartExprNodeDesc(ASTNode astNode,
+ Map<ASTNode, ExprNodeDesc> astExprNodeMap)
+ throws SemanticException, HiveException {
+
+ if ((astNode == null) || (astNode.getChildren() == null) ||
+ (astNode.getChildren().size() <= 1)) {
+ return;
+ }
+
+ TypeCheckCtx typeCheckCtx = new TypeCheckCtx(null);
+ for (Node childNode : astNode.getChildren()) {
+ ASTNode childASTNode = (ASTNode)childNode;
+
+ if (childASTNode.getType() != HiveParser.TOK_PARTVAL) {
+ getPartExprNodeDesc(childASTNode, astExprNodeMap);
+ } else {
+ if (childASTNode.getChildren().size() <= 1) {
+ throw new HiveException("This is dynamic partitioning");
+ }
+
+ ASTNode partValASTChild = (ASTNode)childASTNode.getChildren().get(1);
+ astExprNodeMap.put((ASTNode)childASTNode.getChildren().get(0),
+ TypeCheckProcFactory.genExprNode(partValASTChild, typeCheckCtx).get(partValASTChild));
+ }
+ }
+ }
+
+ public static void validatePartSpec(Table tbl,
+ Map<String, String> partSpec, ASTNode astNode, HiveConf conf) throws SemanticException {
+
+ Map<ASTNode, ExprNodeDesc> astExprNodeMap = new HashMap<ASTNode, ExprNodeDesc>();
+
+ Utilities.validatePartSpec(tbl, partSpec);
+
+ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_TYPE_CHECK_ON_INSERT)) {
+ try {
+ getPartExprNodeDesc(astNode, astExprNodeMap);
+ } catch (HiveException e) {
+ return;
+ }
+ List<FieldSchema> parts = tbl.getPartitionKeys();
+ Map<String, String> partCols = new HashMap<String, String>(parts.size());
+ for (FieldSchema col : parts) {
+ partCols.put(col.getName(), col.getType().toLowerCase());
+ }
+ for (Entry<ASTNode, ExprNodeDesc> astExprNodePair : astExprNodeMap.entrySet()) {
+
+ String astKeyName = astExprNodePair.getKey().toString().toLowerCase();
+ if (astExprNodePair.getKey().getType() == HiveParser.Identifier) {
+ astKeyName = stripIdentifierQuotes(astKeyName);
+ }
+ String colType = partCols.get(astKeyName);
+ ObjectInspector inputOI = astExprNodePair.getValue().getWritableObjectInspector();
+
+ TypeInfo expectedType =
+ TypeInfoUtils.getTypeInfoFromTypeString(colType);
+ ObjectInspector outputOI =
+ TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(expectedType);
+ Object value = null;
+ try {
+ value =
+ ExprNodeEvaluatorFactory.get(astExprNodePair.getValue()).
+ evaluate(partSpec.get(astKeyName));
+ } catch (HiveException e) {
+ throw new SemanticException(e);
+ }
+ Object convertedValue =
+ ObjectInspectorConverters.getConverter(inputOI, outputOI).convert(value);
+ if (convertedValue == null) {
+ throw new SemanticException(ErrorMsg.PARTITION_SPEC_TYPE_MISMATCH.format(astKeyName,
+ inputOI.getTypeName(), outputOI.getTypeName()));
+ }
+ }
+ }
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java Mon Sep 30 21:58:29 2013
@@ -522,6 +522,7 @@ public class ColumnStatsSemanticAnalyzer
boolean isPartitionStats = isPartitionLevelStats(tree);
PartitionList partList = null;
checkForPartitionColumns(colNames, getPartitionKeys(tableName));
+ validateSpecifiedColumnNames(tableName, colNames);
if (isPartitionStats) {
isTableLevel = false;
@@ -545,6 +546,25 @@ public class ColumnStatsSemanticAnalyzer
}
}
+ // fail early if the columns specified for column statistics are not valid
+ private void validateSpecifiedColumnNames(String tableName, List<String> specifiedCols)
+ throws SemanticException {
+ List<FieldSchema> fields = null;
+ try {
+ fields = db.getTable(tableName).getAllCols();
+ } catch (HiveException e) {
+ throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName));
+ }
+ List<String> tableCols = Utilities.getColumnNamesFromFieldSchema(fields);
+
+ for(String sc : specifiedCols) {
+ if (!tableCols.contains(sc.toLowerCase())) {
+ String msg = "'" + sc + "' (possible columns are " + tableCols.toString() + ")";
+ throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(msg));
+ }
+ }
+ }
+
private List<String> getPartitionKeys(String tableName) throws SemanticException {
List<FieldSchema> fields;
try {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Mon Sep 30 21:58:29 2013
@@ -2613,6 +2613,7 @@ public class DDLSemanticAnalyzer extends
currentLocation = null;
}
currentPart = getPartSpec(child);
+ validatePartSpec(tab, currentPart, (ASTNode)child, conf);
break;
case HiveParser.TOK_PARTITIONLOCATION:
// if location specified, set in partition
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java Mon Sep 30 21:58:29 2013
@@ -1097,49 +1097,6 @@ public class PTFTranslator {
return componentInvocations;
}
-
- /*
- * Lead Lag functionality
- */
- public static class LeadLagInfo {
- /*
- * list of LL invocations in a Query.
- */
- List<ExprNodeGenericFuncDesc> leadLagExprs;
- /*
- * map from the Select Expr Node to the LL Function invocations in it.
- */
- Map<ExprNodeDesc, List<ExprNodeGenericFuncDesc>> mapTopExprToLLFunExprs;
-
- private void addLeadLagExpr(ExprNodeGenericFuncDesc llFunc) {
- leadLagExprs = leadLagExprs == null ? new ArrayList<ExprNodeGenericFuncDesc>() : leadLagExprs;
- leadLagExprs.add(llFunc);
- }
-
- public List<ExprNodeGenericFuncDesc> getLeadLagExprs() {
- return leadLagExprs;
- }
-
- public void addLLFuncExprForTopExpr(ExprNodeDesc topExpr, ExprNodeGenericFuncDesc llFuncExpr) {
- addLeadLagExpr(llFuncExpr);
- mapTopExprToLLFunExprs = mapTopExprToLLFunExprs == null ?
- new HashMap<ExprNodeDesc, List<ExprNodeGenericFuncDesc>>() : mapTopExprToLLFunExprs;
- List<ExprNodeGenericFuncDesc> funcList = mapTopExprToLLFunExprs.get(topExpr);
- if (funcList == null) {
- funcList = new ArrayList<ExprNodeGenericFuncDesc>();
- mapTopExprToLLFunExprs.put(topExpr, funcList);
- }
- funcList.add(llFuncExpr);
- }
-
- public List<ExprNodeGenericFuncDesc> getLLFuncExprsInTopExpr(ExprNodeDesc topExpr) {
- if (mapTopExprToLLFunExprs == null) {
- return null;
- }
- return mapTopExprToLLFunExprs.get(topExpr);
- }
- }
-
public static void validateNoLeadLagInValueBoundarySpec(ASTNode node)
throws SemanticException {
String errMsg = "Lead/Lag not allowed in ValueBoundary Spec";
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Mon Sep 30 21:58:29 2013
@@ -110,9 +110,14 @@ public final class TypeCheckProcFactory
// build the exprNodeFuncDesc with recursively built children.
ASTNode expr = (ASTNode) nd;
TypeCheckCtx ctx = (TypeCheckCtx) procCtx;
+
RowResolver input = ctx.getInputRR();
ExprNodeDesc desc = null;
+ if ((ctx == null) || (input == null)) {
+ return null;
+ }
+
// If the current subExpression is pre-calculated, as in Group-By etc.
ColumnInfo colInfo = input.getExpression(expr);
if (colInfo != null) {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingExprNodeEvaluatorFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingExprNodeEvaluatorFactory.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingExprNodeEvaluatorFactory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingExprNodeEvaluatorFactory.java Mon Sep 30 21:58:29 2013
@@ -23,7 +23,6 @@ import java.util.List;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.parse.PTFTranslator.LeadLagInfo;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java Mon Sep 30 21:58:29 2013
@@ -26,9 +26,9 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.PTFUtils;
+import org.apache.hadoop.hive.ql.parse.LeadLagInfo;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputType;
-import org.apache.hadoop.hive.ql.parse.PTFTranslator.LeadLagInfo;
import org.apache.hadoop.hive.ql.parse.RowResolver;
import org.apache.hadoop.hive.ql.parse.TypeCheckCtx;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java Mon Sep 30 21:58:29 2013
@@ -29,7 +29,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.PTFPartition;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.parse.PTFTranslator.LeadLagInfo;
+import org.apache.hadoop.hive.ql.parse.LeadLagInfo;
import org.apache.hadoop.hive.ql.parse.WindowingExprNodeEvaluatorFactory;
import org.apache.hadoop.hive.ql.plan.PTFDesc.BoundaryDef;
import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef;
Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Mon Sep 30 21:58:29 2013
@@ -268,6 +268,11 @@ public class QTestUtil {
String orgScratchDir = conf.getVar(HiveConf.ConfVars.SCRATCHDIR);
conf.setVar(HiveConf.ConfVars.SCRATCHDIR, getHdfsUriString(orgScratchDir));
+
+ if (miniMr) {
+ String orgAuxJarFolder = conf.getAuxJars();
+ conf.setAuxJars(getHdfsUriString("file://" + orgAuxJarFolder));
+ }
}
private String getHdfsUriString(String uriStr) {
@@ -1087,6 +1092,10 @@ public class QTestUtil {
in = new BufferedReader(new FileReader(fname));
out = new BufferedWriter(new FileWriter(fname + ".orig"));
while (null != (line = in.readLine())) {
+ // Ignore the empty lines on windows
+ if(line.isEmpty() && Shell.WINDOWS) {
+ continue;
+ }
out.write(line);
out.write('\n');
}
Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java Mon Sep 30 21:58:29 2013
@@ -41,7 +41,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.io.FSRecordWriter;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.io.InputFormatChecker;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
@@ -521,7 +521,7 @@ public class TestInputOutputFormat {
}
SerDe serde = new OrcSerde();
HiveOutputFormat<?, ?> outFormat = new OrcOutputFormat();
- FileSinkOperator.RecordWriter writer =
+ FSRecordWriter writer =
outFormat.getHiveRecordWriter(conf, testFilePath, MyRow.class, true,
properties, Reporter.NULL);
writer.write(serde.serialize(new MyRow(1,2), inspector));
@@ -686,7 +686,7 @@ public class TestInputOutputFormat {
JobConf job = new JobConf(conf);
Properties properties = new Properties();
HiveOutputFormat<?, ?> outFormat = new OrcOutputFormat();
- FileSinkOperator.RecordWriter writer =
+ FSRecordWriter writer =
outFormat.getHiveRecordWriter(conf, testFilePath, MyRow.class, true,
properties, Reporter.NULL);
writer.close(true);
@@ -731,7 +731,7 @@ public class TestInputOutputFormat {
}
SerDe serde = new OrcSerde();
HiveOutputFormat<?, ?> outFormat = new OrcOutputFormat();
- FileSinkOperator.RecordWriter writer =
+ FSRecordWriter writer =
outFormat.getHiveRecordWriter(conf, testFilePath, StringRow.class,
true, properties, Reporter.NULL);
writer.write(serde.serialize(new StringRow("owen"), inspector));
Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/udf/Rot13OutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/udf/Rot13OutputFormat.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/udf/Rot13OutputFormat.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/udf/Rot13OutputFormat.java Mon Sep 30 21:58:29 2013
@@ -1,7 +1,10 @@
package org.apache.hadoop.hive.ql.io.udf;
+import java.io.IOException;
+import java.util.Properties;
+
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
+import org.apache.hadoop.hive.ql.io.FSRecordWriter;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
@@ -11,27 +14,24 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.Progressable;
-import java.io.IOException;
-import java.util.Properties;
-
public class Rot13OutputFormat
extends HiveIgnoreKeyTextOutputFormat<LongWritable,Text> {
@Override
- public RecordWriter
+ public FSRecordWriter
getHiveRecordWriter(JobConf jc,
Path outPath,
Class<? extends Writable> valueClass,
boolean isCompressed,
Properties tableProperties,
Progressable progress) throws IOException {
- final RecordWriter result =
+ final FSRecordWriter result =
super.getHiveRecordWriter(jc,outPath,valueClass,isCompressed,
tableProperties,progress);
final Reporter reporter = (Reporter) progress;
reporter.setStatus("got here");
System.out.println("Got a reporter " + reporter);
- return new RecordWriter() {
+ return new FSRecordWriter() {
@Override
public void write(Writable w) throws IOException {
if (w instanceof Text) {
Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/udf/TestToInteger.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/udf/TestToInteger.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/udf/TestToInteger.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/udf/TestToInteger.java Mon Sep 30 21:58:29 2013
@@ -25,6 +25,6 @@ public class TestToInteger extends TestC
Text t4 = new Text("1.1");
IntWritable i4 = ti.evaluate(t4);
- assertNull(i4);
+ assertEquals(1, i4.get());
}
}
Modified: hive/branches/vectorization/ql/src/test/queries/clientnegative/deletejar.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientnegative/deletejar.q?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientnegative/deletejar.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientnegative/deletejar.q Mon Sep 30 21:58:29 2013
@@ -1,4 +1,4 @@
-ADD JAR ../data/files/TestSerDe.jar;
-DELETE JAR ../data/files/TestSerDe.jar;
+ADD JAR ../build/ql/test/TestSerDe.jar;
+DELETE JAR ../build/ql/test/TestSerDe.jar;
CREATE TABLE DELETEJAR(KEY STRING, VALUE STRING) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.TestSerDe' STORED AS TEXTFILE;
Modified: hive/branches/vectorization/ql/src/test/queries/clientnegative/invalid_columns.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientnegative/invalid_columns.q?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientnegative/invalid_columns.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientnegative/invalid_columns.q Mon Sep 30 21:58:29 2013
@@ -1,4 +1,4 @@
-ADD JAR ../data/files/TestSerDe.jar;
+ADD JAR ../build/ql/test/TestSerDe.jar;
CREATE TABLE DELETEJAR(KEY STRING, VALUE STRING) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.TestSerDe'
STORED AS TEXTFILE
TBLPROPERTIES('columns'='valid_colname,invalid.colname');
Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/alter1.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/alter1.q?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/alter1.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/alter1.q Mon Sep 30 21:58:29 2013
@@ -15,7 +15,7 @@ describe extended alter1;
alter table alter1 set serdeproperties('s1'='10', 's2' ='20');
describe extended alter1;
-add jar ../data/files/TestSerDe.jar;
+add jar ../build/ql/test/TestSerDe.jar;
alter table alter1 set serde 'org.apache.hadoop.hive.serde2.TestSerDe' with serdeproperties('s1'='9');
describe extended alter1;
@@ -56,7 +56,7 @@ DESCRIBE EXTENDED alter1;
ALTER TABLE alter1 SET SERDEPROPERTIES('s1'='10', 's2' ='20');
DESCRIBE EXTENDED alter1;
-add jar ../data/files/TestSerDe.jar;
+add jar ../build/ql/test/TestSerDe.jar;
ALTER TABLE alter1 SET SERDE 'org.apache.hadoop.hive.serde2.TestSerDe' WITH SERDEPROPERTIES ('s1'='9');
DESCRIBE EXTENDED alter1;
Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/alter_partition_coltype.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/alter_partition_coltype.q?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/alter_partition_coltype.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/alter_partition_coltype.q Mon Sep 30 21:58:29 2013
@@ -24,6 +24,8 @@ select count(*) from alter_coltype where
-- alter partition key column data type for ts column.
alter table alter_coltype partition column (ts double);
+alter table alter_coltype partition column (dt string);
+
-- load a new partition using new data type.
insert overwrite table alter_coltype partition(dt='100x', ts=3.0) select * from src1;
Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/input16.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/input16.q?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/input16.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/input16.q Mon Sep 30 21:58:29 2013
@@ -1,6 +1,6 @@
-- TestSerDe is a user defined serde where the default delimiter is Ctrl-B
DROP TABLE INPUT16;
-ADD JAR ../data/files/TestSerDe.jar;
+ADD JAR ../build/ql/test/TestSerDe.jar;
CREATE TABLE INPUT16(KEY STRING, VALUE STRING) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.TestSerDe' STORED AS TEXTFILE;
LOAD DATA LOCAL INPATH '../data/files/kv1_cb.txt' INTO TABLE INPUT16;
SELECT INPUT16.VALUE, INPUT16.KEY FROM INPUT16;
Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/input16_cc.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/input16_cc.q?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/input16_cc.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/input16_cc.q Mon Sep 30 21:58:29 2013
@@ -4,7 +4,7 @@ set hive.input.format=org.apache.hadoop.
-- the user is overwriting it with ctrlC
DROP TABLE INPUT16_CC;
-ADD JAR ../data/files/TestSerDe.jar;
+ADD JAR ../build/ql/test/TestSerDe.jar;
CREATE TABLE INPUT16_CC(KEY STRING, VALUE STRING) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.TestSerDe' with serdeproperties ('testserde.default.serialization.format'='\003', 'dummy.prop.not.used'='dummyy.val') STORED AS TEXTFILE;
LOAD DATA LOCAL INPATH '../data/files/kv1_cc.txt' INTO TABLE INPUT16_CC;
SELECT INPUT16_CC.VALUE, INPUT16_CC.KEY FROM INPUT16_CC;
Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/reduce_deduplicate_extended.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/reduce_deduplicate_extended.q?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/reduce_deduplicate_extended.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/reduce_deduplicate_extended.q Mon Sep 30 21:58:29 2013
@@ -18,6 +18,7 @@ explain select src.key, sum(src.key) FRO
explain select src.key, src.value FROM src JOIN src1 ON src.key = src1.key order by src.key, src.value;
-- mGBY-RS-rGBY-mGBY-RS-rGBY
explain from (select key, value from src group by key, value) s select s.key group by s.key;
+explain select key, count(distinct value) from (select key, value from src group by key, value) t group by key;
select key, sum(key) from (select * from src distribute by key sort by key, value) Q1 group by key;
select key, sum(key), lower(value) from (select * from src order by key) Q1 group by key, lower(value);
@@ -26,6 +27,7 @@ select key, sum(key) as value from src g
select src.key, sum(src.key) FROM src JOIN src1 ON src.key = src1.key group by src.key, src.value;
select src.key, src.value FROM src JOIN src1 ON src.key = src1.key order by src.key, src.value;
from (select key, value from src group by key, value) s select s.key group by s.key;
+select key, count(distinct value) from (select key, value from src group by key, value) t group by key;
set hive.map.aggr=false;
@@ -41,6 +43,7 @@ explain select src.key, sum(src.key) FRO
explain select src.key, src.value FROM src JOIN src1 ON src.key = src1.key order by src.key, src.value;
-- RS-GBY-RS-GBY
explain from (select key, value from src group by key, value) s select s.key group by s.key;
+explain select key, count(distinct value) from (select key, value from src group by key, value) t group by key;
select key, sum(key) from (select * from src distribute by key sort by key, value) Q1 group by key;
select key, sum(key), lower(value) from (select * from src order by key) Q1 group by key, lower(value);
@@ -49,3 +52,4 @@ select key, sum(key) as value from src g
select src.key, sum(src.key) FROM src JOIN src1 ON src.key = src1.key group by src.key, src.value;
select src.key, src.value FROM src JOIN src1 ON src.key = src1.key order by src.key, src.value;
from (select key, value from src group by key, value) s select s.key group by s.key;
+select key, count(distinct value) from (select key, value from src group by key, value) t group by key;
Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/union_null.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/union_null.q?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/union_null.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/union_null.q Mon Sep 30 21:58:29 2013
@@ -1,2 +1,5 @@
-- HIVE-2901
select x from (select value as x from src union all select NULL as x from src)a limit 10;
+
+-- HIVE-4837
+select * from (select null as N from src1 group by key UNION ALL select null as N from src1 group by key ) a;
Modified: hive/branches/vectorization/ql/src/test/results/beelinepositive/union_null.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/beelinepositive/union_null.q.out?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/beelinepositive/union_null.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/beelinepositive/union_null.q.out Mon Sep 30 21:58:29 2013
@@ -14,4 +14,41 @@ Saving all output to "!!{outputDirectory
'val_165'
''
10 rows selected
+>>>
+>>> -- HIVE-4837
+>>> select * from (select null as N from src1 group by key UNION ALL select null as N from src1 group by key ) a;
+'n'
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+''
+32 rows selected
>>> !record
Modified: hive/branches/vectorization/ql/src/test/results/clientnegative/alter_table_add_partition.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientnegative/alter_table_add_partition.q.out?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientnegative/alter_table_add_partition.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientnegative/alter_table_add_partition.q.out Mon Sep 30 21:58:29 2013
@@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE
POSTHOOK: query: create table mp (a int) partitioned by (b int)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@mp
-FAILED: SemanticException [Error 10214]: Invalid partition spec specified table is partitioned but partition spec is not specified or does not fully match table partitioning: {b=1, c=1}
+FAILED: SemanticException [Error 10098]: Non-Partition column appears in the partition specification: c
Modified: hive/branches/vectorization/ql/src/test/results/clientnegative/alter_view_failure5.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientnegative/alter_view_failure5.q.out?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientnegative/alter_view_failure5.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientnegative/alter_view_failure5.q.out Mon Sep 30 21:58:29 2013
@@ -13,4 +13,4 @@ AS
SELECT * FROM src
POSTHOOK: type: CREATEVIEW
POSTHOOK: Output: default@xxx6
-FAILED: SemanticException [Error 10214]: Invalid partition spec specified value not found in table's partition spec: {v=val_86}
+FAILED: SemanticException [Error 10098]: Non-Partition column appears in the partition specification: v
Modified: hive/branches/vectorization/ql/src/test/results/clientnegative/columnstats_tbllvl.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientnegative/columnstats_tbllvl.q.out?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientnegative/columnstats_tbllvl.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientnegative/columnstats_tbllvl.q.out Mon Sep 30 21:58:29 2013
@@ -33,4 +33,4 @@ PREHOOK: Output: default@uservisits_web_
POSTHOOK: query: LOAD DATA LOCAL INPATH "../data/files/UserVisits.dat" INTO TABLE UserVisits_web_text_none
POSTHOOK: type: LOAD
POSTHOOK: Output: default@uservisits_web_text_none
-FAILED: SemanticException [Error 10004]: Line 1:21 Invalid table alias or column reference 'destIP': (possible column names are: sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite)
+FAILED: SemanticException [Error 10002]: Invalid column reference 'destIP' (possible columns are [sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite])
Modified: hive/branches/vectorization/ql/src/test/results/clientnegative/columnstats_tbllvl_incorrect_column.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientnegative/columnstats_tbllvl_incorrect_column.q.out?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientnegative/columnstats_tbllvl_incorrect_column.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientnegative/columnstats_tbllvl_incorrect_column.q.out Mon Sep 30 21:58:29 2013
@@ -33,4 +33,4 @@ PREHOOK: Output: default@uservisits_web_
POSTHOOK: query: LOAD DATA LOCAL INPATH "../data/files/UserVisits.dat" INTO TABLE UserVisits_web_text_none
POSTHOOK: type: LOAD
POSTHOOK: Output: default@uservisits_web_text_none
-FAILED: SemanticException [Error 10004]: Line 1:21 Invalid table alias or column reference 'destIP': (possible column names are: sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite)
+FAILED: SemanticException [Error 10002]: Invalid column reference 'destIP' (possible columns are [sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite])
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/alter_partition_coltype.q.out?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/alter_partition_coltype.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/alter_partition_coltype.q.out Mon Sep 30 21:58:29 2013
@@ -242,6 +242,17 @@ POSTHOOK: Lineage: alter_coltype PARTITI
POSTHOOK: Lineage: alter_coltype PARTITION(dt=10,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: alter table alter_coltype partition column (dt string)
+PREHOOK: type: null
+PREHOOK: Input: default@alter_coltype
+POSTHOOK: query: alter table alter_coltype partition column (dt string)
+POSTHOOK: type: null
+POSTHOOK: Input: default@alter_coltype
+POSTHOOK: Output: default@alter_coltype
+POSTHOOK: Lineage: alter_coltype PARTITION(dt=10,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: alter_coltype PARTITION(dt=10,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: -- load a new partition using new data type.
insert overwrite table alter_coltype partition(dt='100x', ts=3.0) select * from src1
PREHOOK: type: QUERY
@@ -1239,13 +1250,13 @@ POSTHOOK: Lineage: alter_coltype PARTITI
POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
key string None
value string None
-dt int None
+dt string None
ts double None
# Partition Information
# col_name data_type comment
-dt int None
+dt string None
ts double None
PREHOOK: query: desc alter_coltype partition (dt='100x', ts='6:30pm')
PREHOOK: type: DESCTABLE
@@ -1259,13 +1270,13 @@ POSTHOOK: Lineage: alter_coltype PARTITI
POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
key string None
value string None
-dt int None
+dt string None
ts double None
# Partition Information
# col_name data_type comment
-dt int None
+dt string None
ts double None
PREHOOK: query: desc alter_coltype partition (dt='100x', ts=3.0)
PREHOOK: type: DESCTABLE
@@ -1279,13 +1290,13 @@ POSTHOOK: Lineage: alter_coltype PARTITI
POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
key string None
value string None
-dt int None
+dt string None
ts double None
# Partition Information
# col_name data_type comment
-dt int None
+dt string None
ts double None
PREHOOK: query: desc alter_coltype partition (dt=10, ts=3.0)
PREHOOK: type: DESCTABLE
@@ -1299,13 +1310,13 @@ POSTHOOK: Lineage: alter_coltype PARTITI
POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
key string None
value string None
-dt int None
+dt string None
ts double None
# Partition Information
# col_name data_type comment
-dt int None
+dt string None
ts double None
PREHOOK: query: drop table alter_coltype
PREHOOK: type: DROPTABLE
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/reduce_deduplicate_extended.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/reduce_deduplicate_extended.q.out?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/reduce_deduplicate_extended.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/reduce_deduplicate_extended.q.out Mon Sep 30 21:58:29 2013
@@ -659,6 +659,97 @@ STAGE PLANS:
limit: -1
+PREHOOK: query: explain select key, count(distinct value) from (select key, value from src group by key, value) t group by key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select key, count(distinct value) from (select key, value from src group by key, value) t group by key
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)))) t)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTIONDI count (TOK_TABLE_OR_COL value)))) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ t:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ Reduce Operator Tree:
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ expr: KEY._col1
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations:
+ expr: count(DISTINCT _col1)
+ bucketGroup: false
+ keys:
+ expr: _col0
+ type: string
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
PREHOOK: query: select key, sum(key) from (select * from src distribute by key sort by key, value) Q1 group by key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
@@ -2316,6 +2407,323 @@ POSTHOOK: Input: default@src
96
97
98
+PREHOOK: query: select key, count(distinct value) from (select key, value from src group by key, value) t group by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key, count(distinct value) from (select key, value from src group by key, value) t group by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 1
+10 1
+100 1
+103 1
+104 1
+105 1
+11 1
+111 1
+113 1
+114 1
+116 1
+118 1
+119 1
+12 1
+120 1
+125 1
+126 1
+128 1
+129 1
+131 1
+133 1
+134 1
+136 1
+137 1
+138 1
+143 1
+145 1
+146 1
+149 1
+15 1
+150 1
+152 1
+153 1
+155 1
+156 1
+157 1
+158 1
+160 1
+162 1
+163 1
+164 1
+165 1
+166 1
+167 1
+168 1
+169 1
+17 1
+170 1
+172 1
+174 1
+175 1
+176 1
+177 1
+178 1
+179 1
+18 1
+180 1
+181 1
+183 1
+186 1
+187 1
+189 1
+19 1
+190 1
+191 1
+192 1
+193 1
+194 1
+195 1
+196 1
+197 1
+199 1
+2 1
+20 1
+200 1
+201 1
+202 1
+203 1
+205 1
+207 1
+208 1
+209 1
+213 1
+214 1
+216 1
+217 1
+218 1
+219 1
+221 1
+222 1
+223 1
+224 1
+226 1
+228 1
+229 1
+230 1
+233 1
+235 1
+237 1
+238 1
+239 1
+24 1
+241 1
+242 1
+244 1
+247 1
+248 1
+249 1
+252 1
+255 1
+256 1
+257 1
+258 1
+26 1
+260 1
+262 1
+263 1
+265 1
+266 1
+27 1
+272 1
+273 1
+274 1
+275 1
+277 1
+278 1
+28 1
+280 1
+281 1
+282 1
+283 1
+284 1
+285 1
+286 1
+287 1
+288 1
+289 1
+291 1
+292 1
+296 1
+298 1
+30 1
+302 1
+305 1
+306 1
+307 1
+308 1
+309 1
+310 1
+311 1
+315 1
+316 1
+317 1
+318 1
+321 1
+322 1
+323 1
+325 1
+327 1
+33 1
+331 1
+332 1
+333 1
+335 1
+336 1
+338 1
+339 1
+34 1
+341 1
+342 1
+344 1
+345 1
+348 1
+35 1
+351 1
+353 1
+356 1
+360 1
+362 1
+364 1
+365 1
+366 1
+367 1
+368 1
+369 1
+37 1
+373 1
+374 1
+375 1
+377 1
+378 1
+379 1
+382 1
+384 1
+386 1
+389 1
+392 1
+393 1
+394 1
+395 1
+396 1
+397 1
+399 1
+4 1
+400 1
+401 1
+402 1
+403 1
+404 1
+406 1
+407 1
+409 1
+41 1
+411 1
+413 1
+414 1
+417 1
+418 1
+419 1
+42 1
+421 1
+424 1
+427 1
+429 1
+43 1
+430 1
+431 1
+432 1
+435 1
+436 1
+437 1
+438 1
+439 1
+44 1
+443 1
+444 1
+446 1
+448 1
+449 1
+452 1
+453 1
+454 1
+455 1
+457 1
+458 1
+459 1
+460 1
+462 1
+463 1
+466 1
+467 1
+468 1
+469 1
+47 1
+470 1
+472 1
+475 1
+477 1
+478 1
+479 1
+480 1
+481 1
+482 1
+483 1
+484 1
+485 1
+487 1
+489 1
+490 1
+491 1
+492 1
+493 1
+494 1
+495 1
+496 1
+497 1
+498 1
+5 1
+51 1
+53 1
+54 1
+57 1
+58 1
+64 1
+65 1
+66 1
+67 1
+69 1
+70 1
+72 1
+74 1
+76 1
+77 1
+78 1
+8 1
+80 1
+82 1
+83 1
+84 1
+85 1
+86 1
+87 1
+9 1
+90 1
+92 1
+95 1
+96 1
+97 1
+98 1
PREHOOK: query: -- RS-RS-GBY
explain select key, sum(key) from (select * from src distribute by key sort by key, value) Q1 group by key
PREHOOK: type: QUERY
@@ -2934,6 +3342,88 @@ STAGE PLANS:
limit: -1
+PREHOOK: query: explain select key, count(distinct value) from (select key, value from src group by key, value) t group by key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select key, count(distinct value) from (select key, value from src group by key, value) t group by key
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)))) t)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_FUNCTIONDI count (TOK_TABLE_OR_COL value)))) (TOK_GROUPBY (TOK_TABLE_OR_COL key))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ t:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: -1
+ Reduce Operator Tree:
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ expr: KEY._col1
+ type: string
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations:
+ expr: count(DISTINCT _col1)
+ bucketGroup: false
+ keys:
+ expr: _col0
+ type: string
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
PREHOOK: query: select key, sum(key) from (select * from src distribute by key sort by key, value) Q1 group by key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
@@ -4591,3 +5081,320 @@ POSTHOOK: Input: default@src
96
97
98
+PREHOOK: query: select key, count(distinct value) from (select key, value from src group by key, value) t group by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key, count(distinct value) from (select key, value from src group by key, value) t group by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 1
+10 1
+100 1
+103 1
+104 1
+105 1
+11 1
+111 1
+113 1
+114 1
+116 1
+118 1
+119 1
+12 1
+120 1
+125 1
+126 1
+128 1
+129 1
+131 1
+133 1
+134 1
+136 1
+137 1
+138 1
+143 1
+145 1
+146 1
+149 1
+15 1
+150 1
+152 1
+153 1
+155 1
+156 1
+157 1
+158 1
+160 1
+162 1
+163 1
+164 1
+165 1
+166 1
+167 1
+168 1
+169 1
+17 1
+170 1
+172 1
+174 1
+175 1
+176 1
+177 1
+178 1
+179 1
+18 1
+180 1
+181 1
+183 1
+186 1
+187 1
+189 1
+19 1
+190 1
+191 1
+192 1
+193 1
+194 1
+195 1
+196 1
+197 1
+199 1
+2 1
+20 1
+200 1
+201 1
+202 1
+203 1
+205 1
+207 1
+208 1
+209 1
+213 1
+214 1
+216 1
+217 1
+218 1
+219 1
+221 1
+222 1
+223 1
+224 1
+226 1
+228 1
+229 1
+230 1
+233 1
+235 1
+237 1
+238 1
+239 1
+24 1
+241 1
+242 1
+244 1
+247 1
+248 1
+249 1
+252 1
+255 1
+256 1
+257 1
+258 1
+26 1
+260 1
+262 1
+263 1
+265 1
+266 1
+27 1
+272 1
+273 1
+274 1
+275 1
+277 1
+278 1
+28 1
+280 1
+281 1
+282 1
+283 1
+284 1
+285 1
+286 1
+287 1
+288 1
+289 1
+291 1
+292 1
+296 1
+298 1
+30 1
+302 1
+305 1
+306 1
+307 1
+308 1
+309 1
+310 1
+311 1
+315 1
+316 1
+317 1
+318 1
+321 1
+322 1
+323 1
+325 1
+327 1
+33 1
+331 1
+332 1
+333 1
+335 1
+336 1
+338 1
+339 1
+34 1
+341 1
+342 1
+344 1
+345 1
+348 1
+35 1
+351 1
+353 1
+356 1
+360 1
+362 1
+364 1
+365 1
+366 1
+367 1
+368 1
+369 1
+37 1
+373 1
+374 1
+375 1
+377 1
+378 1
+379 1
+382 1
+384 1
+386 1
+389 1
+392 1
+393 1
+394 1
+395 1
+396 1
+397 1
+399 1
+4 1
+400 1
+401 1
+402 1
+403 1
+404 1
+406 1
+407 1
+409 1
+41 1
+411 1
+413 1
+414 1
+417 1
+418 1
+419 1
+42 1
+421 1
+424 1
+427 1
+429 1
+43 1
+430 1
+431 1
+432 1
+435 1
+436 1
+437 1
+438 1
+439 1
+44 1
+443 1
+444 1
+446 1
+448 1
+449 1
+452 1
+453 1
+454 1
+455 1
+457 1
+458 1
+459 1
+460 1
+462 1
+463 1
+466 1
+467 1
+468 1
+469 1
+47 1
+470 1
+472 1
+475 1
+477 1
+478 1
+479 1
+480 1
+481 1
+482 1
+483 1
+484 1
+485 1
+487 1
+489 1
+490 1
+491 1
+492 1
+493 1
+494 1
+495 1
+496 1
+497 1
+498 1
+5 1
+51 1
+53 1
+54 1
+57 1
+58 1
+64 1
+65 1
+66 1
+67 1
+69 1
+70 1
+72 1
+74 1
+76 1
+77 1
+78 1
+8 1
+80 1
+82 1
+83 1
+84 1
+85 1
+86 1
+87 1
+9 1
+90 1
+92 1
+95 1
+96 1
+97 1
+98 1
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/union_null.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/union_null.q.out?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/union_null.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/union_null.q.out Mon Sep 30 21:58:29 2013
@@ -18,3 +18,45 @@ val_27
NULL
val_165
NULL
+PREHOOK: query: -- HIVE-4837
+select * from (select null as N from src1 group by key UNION ALL select null as N from src1 group by key ) a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src1
+#### A masked pattern was here ####
+POSTHOOK: query: -- HIVE-4837
+select * from (select null as N from src1 group by key UNION ALL select null as N from src1 group by key ) a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src1
+#### A masked pattern was here ####
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java Mon Sep 30 21:58:29 2013
@@ -27,9 +27,11 @@ public class SerDeStats {
// currently we support only raw data size stat
private long rawDataSize;
+ private long rowCount;
public SerDeStats() {
rawDataSize = 0;
+ rowCount = 0;
}
/**
@@ -48,4 +50,20 @@ public class SerDeStats {
rawDataSize = uSize;
}
+ /**
+ * Return the row count
+ * @return row count
+ */
+ public long getRowCount() {
+ return rowCount;
+ }
+
+ /**
+ * Set the row count
+ * @param rowCount - count of rows
+ */
+ public void setRowCount(long rowCount) {
+ this.rowCount = rowCount;
+ }
+
}
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java Mon Sep 30 21:58:29 2013
@@ -136,11 +136,18 @@ public class LazyInteger extends
*/
private static int parse(byte[] bytes, int start, int length, int offset,
int radix, boolean negative) {
+ byte separator = '.';
int max = Integer.MIN_VALUE / radix;
int result = 0, end = start + length;
while (offset < end) {
int digit = LazyUtils.digit(bytes[offset++], radix);
if (digit == -1) {
+ if (bytes[offset-1] == separator) {
+ // We allow decimals and will return a truncated integer in that case.
+ // Therefore we won't throw an exception here (checking the fractional
+ // part happens below.)
+ break;
+ }
throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
length));
}
@@ -155,6 +162,18 @@ public class LazyInteger extends
}
result = next;
}
+
+ // This is the case when we've encountered a decimal separator. The fractional
+ // part will not change the number, but we will verify that the fractional part
+ // is well formed.
+ while (offset < end) {
+ int digit = LazyUtils.digit(bytes[offset++], radix);
+ if (digit == -1) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
+ }
+ }
+
if (!negative) {
result = -result;
if (result < 0) {
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java Mon Sep 30 21:58:29 2013
@@ -138,11 +138,18 @@ public class LazyLong extends
*/
private static long parse(byte[] bytes, int start, int length, int offset,
int radix, boolean negative) {
+ byte separator = '.';
long max = Long.MIN_VALUE / radix;
long result = 0, end = start + length;
while (offset < end) {
int digit = LazyUtils.digit(bytes[offset++], radix);
if (digit == -1 || max > result) {
+ if (bytes[offset-1] == separator) {
+ // We allow decimals and will return a truncated integer in that case.
+ // Therefore we won't throw an exception here (checking the fractional
+ // part happens below.)
+ break;
+ }
throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
length));
}
@@ -153,6 +160,18 @@ public class LazyLong extends
}
result = next;
}
+
+ // This is the case when we've encountered a decimal separator. The fractional
+ // part will not change the number, but we will verify that the fractional part
+ // is well formed.
+ while (offset < end) {
+ int digit = LazyUtils.digit(bytes[offset++], radix);
+ if (digit == -1) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
+ }
+ }
+
if (!negative) {
result = -result;
if (result < 0) {
Modified: hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java (original)
+++ hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java Mon Sep 30 21:58:29 2013
@@ -62,12 +62,12 @@ public class TestLazySimpleSerDe extends
// Data
Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\tNULL\t");
t.append(new byte[]{(byte)Integer.parseInt("10111111", 2)}, 0, 1);
- StringBuffer sb = new StringBuffer("123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\tNULL\t");
+ StringBuffer sb = new StringBuffer("123\t456\t789\t1000\t5.3\thive and hadoop\t1\tNULL\t");
String s = sb.append(new String(Base64.encodeBase64(new byte[]{(byte)Integer.parseInt("10111111", 2)}))).toString();
Object[] expectedFieldsData = {new ByteWritable((byte) 123),
new ShortWritable((short) 456), new IntWritable(789),
new LongWritable(1000), new DoubleWritable(5.3),
- new Text("hive and hadoop"), null, null, new BytesWritable(new byte[]{(byte)Integer.parseInt("10111111", 2)})};
+ new Text("hive and hadoop"), new IntWritable(1), null, new BytesWritable(new byte[]{(byte)Integer.parseInt("10111111", 2)})};
// Test
deserializeAndSerialize(serDe, t, s, expectedFieldsData);
@@ -128,11 +128,11 @@ public class TestLazySimpleSerDe extends
// Data
Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\ta\tb\t");
- String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\ta\tb\t";
+ String s = "123\t456\t789\t1000\t5.3\thive and hadoop\t1\ta\tb\t";
Object[] expectedFieldsData = {new ByteWritable((byte) 123),
new ShortWritable((short) 456), new IntWritable(789),
new LongWritable(1000), new DoubleWritable(5.3),
- new Text("hive and hadoop"), null, new Text("a\tb\t")};
+ new Text("hive and hadoop"), new IntWritable(1), new Text("a\tb\t")};
// Test
deserializeAndSerialize(serDe, t, s, expectedFieldsData);
@@ -156,11 +156,11 @@ public class TestLazySimpleSerDe extends
// Data
Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\ta\tb\t");
- String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\ta";
+ String s = "123\t456\t789\t1000\t5.3\thive and hadoop\t1\ta";
Object[] expectedFieldsData = {new ByteWritable((byte) 123),
new ShortWritable((short) 456), new IntWritable(789),
new LongWritable(1000), new DoubleWritable(5.3),
- new Text("hive and hadoop"), null, new Text("a")};
+ new Text("hive and hadoop"), new IntWritable(1), new Text("a")};
// Test
deserializeAndSerialize(serDe, t, s, expectedFieldsData);
Modified: hive/branches/vectorization/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java (original)
+++ hive/branches/vectorization/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java Mon Sep 30 21:58:29 2013
@@ -110,6 +110,7 @@ public class PTest {
put("buildTag", buildTag).
put("logDir", logDir.getAbsolutePath()).
put("javaHome", configuration.getJavaHome()).
+ put("javaHomeForTests", configuration.getJavaHomeForTests()).
put("antEnvOpts", configuration.getAntEnvOpts());
final ImmutableMap<String, String> templateDefaults = templateDefaultsBuilder.build();
TestParser testParser = new TestParser(configuration.getContext(),
@@ -221,12 +222,13 @@ public class PTest {
}
private static final String PROPERTIES = "properties";
- private static final String REPOSITORY = "repository";
- private static final String REPOSITORY_NAME = "repositoryName";
- private static final String BRANCH = "branch";
+ private static final String REPOSITORY = TestConfiguration.REPOSITORY;
+ private static final String REPOSITORY_NAME = TestConfiguration.REPOSITORY_NAME;
+ private static final String BRANCH = TestConfiguration.BRANCH;
private static final String PATCH = "patch";
- private static final String JAVA_HOME = "javaHome";
- private static final String ANT_ENV_OPTS = "antEnvOpts";
+ private static final String JAVA_HOME = TestConfiguration.JAVA_HOME;
+ private static final String JAVA_HOME_TEST = TestConfiguration.JAVA_HOME_TEST;
+ private static final String ANT_ENV_OPTS = TestConfiguration.ANT_ENV_OPTS;
/**
* All args override properties file settings except
* for this one which is additive.
@@ -243,7 +245,8 @@ public class PTest {
options.addOption(null, BRANCH, true, "Overrides git branch in properties file");
options.addOption(null, PATCH, true, "URI to patch, either file:/// or http(s)://");
options.addOption(ANT_ARG, null, true, "Supplemntal ant arguments");
- options.addOption(null, JAVA_HOME, true, "Java Home for compiling and running tests");
+ options.addOption(null, JAVA_HOME, true, "Java Home for compiling and running tests (unless " + JAVA_HOME_TEST + " is specified)");
+ options.addOption(null, JAVA_HOME_TEST, true, "Java Home for running tests (optional)");
options.addOption(null, ANT_ENV_OPTS, true, "ANT_OPTS environemnt variable setting");
CommandLine commandLine = parser.parse(options, args);
if(!commandLine.hasOption(PROPERTIES)) {
@@ -282,6 +285,10 @@ public class PTest {
if(!javaHome.isEmpty()) {
conf.setJavaHome(javaHome);
}
+ String javaHomeForTests = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME_TEST)).trim();
+ if(!javaHomeForTests.isEmpty()) {
+ conf.setJavaHomeForTests(javaHomeForTests);
+ }
String antEnvOpts = Strings.nullToEmpty(commandLine.getOptionValue(ANT_ENV_OPTS)).trim();
if(!antEnvOpts.isEmpty()) {
conf.setAntEnvOpts(antEnvOpts);
Modified: hive/branches/vectorization/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java?rev=1527793&r1=1527792&r2=1527793&view=diff
==============================================================================
--- hive/branches/vectorization/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java (original)
+++ hive/branches/vectorization/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java Mon Sep 30 21:58:29 2013
@@ -32,19 +32,21 @@ import com.google.common.base.Strings;
import com.google.common.collect.Maps;
public class TestConfiguration {
+ public static final String REPOSITORY = "repository";
+ public static final String REPOSITORY_NAME = "repositoryName";
+ public static final String BRANCH = "branch";
+ public static final String JAVA_HOME = "javaHome";
+ public static final String JAVA_HOME_TEST = "javaHomeForTests";
+ public static final String ANT_ENV_OPTS = "antEnvOpts";
+
private static final String REPOSITORY_TYPE = "repositoryType";
private static final String GIT = "git";
private static final String SVN = "svn";
- private static final String REPOSITORY = "repository";
- private static final String REPOSITORY_NAME = "repositoryName";
- private static final String BRANCH = "branch";
private static final String ANT_ARGS = "antArgs";
- private static final String ANT_ENV_OPTS = "antEnvOpts";
- private static final String JAVA_HOME = "javaHome";
private static final String JIRA_URL = "jiraUrl";
private static final String JIRA_USER = "jiraUser";
private static final String JIRA_PASSWORD = "jiraPassword";
- public static final String JENKINS_URL = "jenkinsURL";
+ private static final String JENKINS_URL = "jenkinsURL";
private final Context context;
private String antArgs;
@@ -54,6 +56,7 @@ public class TestConfiguration {
private String repositoryName;
private String patch;
private String javaHome;
+ private String javaHomeForTests;
private String branch;
private final String jenkinsURL;
private final String jiraUrl;
@@ -79,6 +82,7 @@ public class TestConfiguration {
antArgs = Preconditions.checkNotNull(context.getString(ANT_ARGS), ANT_ARGS).trim();
antEnvOpts = context.getString(ANT_ENV_OPTS, "").trim();
javaHome = context.getString(JAVA_HOME, "").trim();
+ javaHomeForTests = context.getString(JAVA_HOME_TEST, "").trim();
patch = Strings.nullToEmpty(null);
jiraName = Strings.nullToEmpty(null);
jiraUrl = context.getString(JIRA_URL, "").trim();
@@ -132,6 +136,9 @@ public class TestConfiguration {
public String getJavaHome() {
return javaHome;
}
+ public String getJavaHomeForTests() {
+ return javaHomeForTests;
+ }
public String getPatch() {
return patch;
}
@@ -150,6 +157,9 @@ public class TestConfiguration {
public void setJavaHome(String javaHome) {
this.javaHome = Strings.nullToEmpty(javaHome);
}
+ public void setJavaHomeForTests(String javaHomeForTests) {
+ this.javaHomeForTests = javaHomeForTests;
+ }
public void setAntArgs(String antArgs) {
this.antArgs = Strings.nullToEmpty(antArgs);
}
@@ -161,10 +171,14 @@ public class TestConfiguration {
}
@Override
public String toString() {
- return "Configuration [context=" + context + ", antArgs=" + antArgs
- + ", antEnvOpts=" + antEnvOpts + ", repository=" + repository
- + ", repositoryName=" + repositoryName + ", patch=" + patch
- + ", javaHome=" + javaHome + ", branch=" + branch + "]";
+ return "TestConfiguration [antArgs=" + antArgs + ", antEnvOpts="
+ + antEnvOpts + ", repositoryType=" + repositoryType + ", repository="
+ + repository + ", repositoryName=" + repositoryName + ", patch="
+ + patch + ", javaHome=" + javaHome + ", javaHomeForTests="
+ + javaHomeForTests + ", branch=" + branch + ", jenkinsURL="
+ + jenkinsURL + ", jiraUrl=" + jiraUrl + ", jiraUser=" + jiraUser
+ + ", jiraName=" + jiraName + ", clearLibraryCache=" + clearLibraryCache
+ + "]";
}
public static TestConfiguration fromInputStream(InputStream inputStream, Logger logger)
throws IOException {