You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/08/05 23:06:25 UTC
svn commit: r1510732 [1/10] - in /hive/branches/vectorization: ./
ql/src/java/org/apache/hadoop/hive/ql/history/
ql/src/test/org/apache/hadoop/hive/ql/
ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/
ql/src/test/queries/clientpositive/ ql/src/t...
Author: hashutosh
Date: Mon Aug 5 21:06:24 2013
New Revision: 1510732
URL: http://svn.apache.org/r1510732
Log:
HIVE-4794 : Unit e2e tests for vectorization (Tony Murphy via Ashutosh Chauhan)
Added:
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/AllVectorTypesRecord.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/OrcFileGenerator.java
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_0.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_1.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_10.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_11.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_12.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_13.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_14.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_15.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_16.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_2.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_3.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_4.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_5.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_6.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_7.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_8.q
hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_9.q
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_0.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_10.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_11.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_12.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_13.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_14.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_15.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_16.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_5.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_6.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_7.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_8.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_9.q.out
Modified:
hive/branches/vectorization/.gitignore
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
hive/branches/vectorization/ql/src/test/results/clientpositive/add_part_exist.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/alter1.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/alter2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/alter3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/alter4.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/alter5.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/alter_index.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/alter_rename_partition.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/describe_table_json.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/index_creation.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input2.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/input3.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/rename_column.q.out
hive/branches/vectorization/ql/src/test/results/clientpositive/show_tables.q.out
Modified: hive/branches/vectorization/.gitignore
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/.gitignore?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/.gitignore (original)
+++ hive/branches/vectorization/.gitignore Mon Aug 5 21:06:24 2013
@@ -17,3 +17,4 @@ ql/derby.log
derby.log
.arc
ql/TempStatsStore
+data/files/*alltypesorc*
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java Mon Aug 5 21:06:24 2013
@@ -287,7 +287,7 @@ public class HiveHistory {
String key = ent.getKey();
String val = ent.getValue();
if(val != null) {
- val = val.replace(System.getProperty("line.separator"), " ");
+ val = val.replace("\r","").replace("\n", " ");
}
sb.append(key + "=\"" + val + "\"");
Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Mon Aug 5 21:06:24 2013
@@ -63,6 +63,8 @@ import org.apache.hadoop.hive.ql.exec.Fu
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.Utilities.StreamPrinter;
+import org.apache.hadoop.hive.ql.exec.vector.util.AllVectorTypesRecord;
+import org.apache.hadoop.hive.ql.exec.vector.util.OrcFileGenerator;
import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -105,7 +107,8 @@ public class QTestUtil {
public static final HashSet<String> srcTables = new HashSet<String>
(Arrays.asList(new String [] {
"src", "src1", "srcbucket", "srcbucket2", "src_json", "src_thrift",
- "src_sequencefile", "srcpart"
+ "src_sequencefile", "srcpart",
+ AllVectorTypesRecord.TABLE_NAME
}));
private ParseDriver pd;
@@ -515,7 +518,8 @@ public class QTestUtil {
for (String s : new String[] {"src", "src1", "src_json", "src_thrift",
"src_sequencefile", "srcpart", "srcbucket", "srcbucket2", "dest1",
"dest2", "dest3", "dest4", "dest4_sequencefile", "dest_j1", "dest_j2",
- "dest_g1", "dest_g2", "fetchtask_ioexception"}) {
+ "dest_g1", "dest_g2", "fetchtask_ioexception",
+ AllVectorTypesRecord.TABLE_NAME}) {
db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, s);
}
@@ -656,7 +660,22 @@ public class QTestUtil {
fpath = new Path(testFiles, "json.txt");
runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
+ "' INTO TABLE src_json");
+
+ FileSystem localFs = FileSystem.getLocal(conf);
+ // create and load data into orc table
+ fpath = new Path(testFiles, AllVectorTypesRecord.TABLE_NAME);
+
+ localFs.delete(fpath, false);
+
+ OrcFileGenerator.generateOrcFile(conf, localFs, fpath,
+ AllVectorTypesRecord.class);
+
+ runCreateTableCmd(AllVectorTypesRecord.TABLE_CREATE_COMMAND);
+ runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath()
+ + "' INTO TABLE "+AllVectorTypesRecord.TABLE_NAME);
+
conf.setBoolean("hive.test.init.phase", false);
+
}
public void init() throws Exception {
Added: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/AllVectorTypesRecord.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/AllVectorTypesRecord.java?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/AllVectorTypesRecord.java (added)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/AllVectorTypesRecord.java Mon Aug 5 21:06:24 2013
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.util;
+
+import java.sql.Timestamp;
+
+/**
+ *
+ * AllTypesRecord.
+ *
+ */
+public class AllVectorTypesRecord {
+ private final Byte ctinyInt;
+ private final Short csmallInt;
+ private final Integer cint;
+ private final Long cbigInt;
+
+ private final Float cfloat;
+ private final Double cdouble;
+
+ private final String cstring1;
+ private final String cstring2;
+
+ private final Timestamp ctimestamp1;
+ private final Timestamp ctimestamp2;
+
+ private final Boolean cboolean1;
+ private final Boolean cboolean2;
+
+ /**
+ *
+ * @param ctinyInt
+ * @param csmallInt
+ * @param cint
+ * @param cbigInt
+ * @param cfloat
+ * @param cdouble
+ * @param cstring1
+ * @param cstring2
+ * @param ctimestamp1
+ * @param ctimestamp2
+ * @param cboolean1
+ * @param cboolean2
+ */
+ public AllVectorTypesRecord(Byte ctinyInt, Short csmallInt, Integer cint, Long cbigInt,
+ Float cfloat, Double cdouble, String cstring1, String cstring2, Timestamp ctimestamp1,
+ Timestamp ctimestamp2, Boolean cboolean1, Boolean cboolean2) {
+
+ this.ctinyInt = ctinyInt;
+ this.csmallInt = csmallInt;
+ this.cint = cint;
+ this.cbigInt = cbigInt;
+
+ this.cfloat = cfloat;
+ this.cdouble = cdouble;
+
+ this.cstring1 = cstring1;
+ this.cstring2 = cstring2;
+
+ this.ctimestamp1 = ctimestamp1;
+ this.ctimestamp2 = ctimestamp2;
+
+ this.cboolean1 = cboolean1;
+ this.cboolean2 = cboolean2;
+ }
+
+ public static final String TABLE_NAME = "alltypesorc";
+
+ public static final String TABLE_CREATE_COMMAND =
+ "CREATE TABLE " + TABLE_NAME + "(" +
+ "ctinyint tinyint, " +
+ "csmallint smallint, " +
+ "cint int, " +
+ "cbigint bigint, " +
+ "cfloat float, " +
+ "cdouble double, " +
+ "cstring1 string, " +
+ "cstring2 string, " +
+ "ctimestamp1 timestamp, " +
+ "ctimestamp2 timestamp, " +
+ "cboolean1 boolean, " +
+ "cboolean2 boolean) " +
+ "STORED AS ORC";
+}
Added: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/OrcFileGenerator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/OrcFileGenerator.java?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/OrcFileGenerator.java (added)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/OrcFileGenerator.java Mon Aug 5 21:06:24 2013
@@ -0,0 +1,355 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.util;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.sql.Timestamp;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
+import org.apache.hadoop.hive.ql.io.orc.OrcFile;
+import org.apache.hadoop.hive.ql.io.orc.TestVectorizedORCReader;
+import org.apache.hadoop.hive.ql.io.orc.Writer;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+
+/**
+ * This class generates an orc file from a specified record class. The orc file
+ * will contain 3 batches worth of rows for each column for all kinds of data distribution:
+ * all values, no nulls, repeating value, and repeating null.
+ *
+ */
+public class OrcFileGenerator {
+
+ enum BatchDataDistribution {
+ AllValues,
+ NoNulls,
+ RepeatingValue,
+ RepeatingNull
+ }
+
+ /**
+ * Base class for type specific batch generators. Each type specific batch generator implements
+ * generateRandomNonNullValue to generate random values, and initializeFixedPointValues to
+ * specify a set of fixed values within the data (this is useful when defining query predicates)
+ */
+ private abstract static class BatchGenerator<T> {
+
+ private final Random rand = new Random(0xfa57);
+ private int possibleNonRandomValueGenerated = rand.nextInt();
+ private final T[] fixedPointValues;
+
+ public BatchGenerator() {
+ fixedPointValues = initializeFixedPointValues();
+ }
+
+ protected abstract T[] initializeFixedPointValues();
+
+ protected abstract T generateRandomNonNullValue(Random rand);
+
+ public T[] generateBatch(BatchDataDistribution dist) {
+
+ Object[] batch = new Object[VectorizedRowBatch.DEFAULT_SIZE];
+
+ for (int i = 0; i < batch.length; i++) {
+ switch (dist) {
+ case AllValues:
+ if (possibleNonRandomValueGenerated % 73 == 0) {
+ batch[i] = null;
+ } else if (fixedPointValues != null && possibleNonRandomValueGenerated % 233 == 0) {
+ batch[i] = fixedPointValues[rand.nextInt(fixedPointValues.length)];
+ } else {
+ batch[i] = generateRandomNonNullValue(rand);
+ }
+ possibleNonRandomValueGenerated++;
+ break;
+
+ case NoNulls:
+ if (fixedPointValues != null && possibleNonRandomValueGenerated % 233 == 0) {
+ batch[i] = fixedPointValues[rand.nextInt(fixedPointValues.length)];
+ } else {
+ batch[i] = generateRandomNonNullValue(rand);
+ }
+ possibleNonRandomValueGenerated++;
+ break;
+
+ case RepeatingNull:
+ batch[i] = null;
+ break;
+
+ case RepeatingValue:
+ if (i == 0) {
+ batch[i] = generateRandomNonNullValue(rand);
+ } else {
+ batch[i] = batch[0];
+ }
+ break;
+
+ default:
+ throw new UnsupportedOperationException(
+ dist.toString() + " data distribution is not implemented.");
+ }
+ }
+
+ return (T[]) batch;
+ }
+ }
+
+ private static class ByteBatchGenerator extends BatchGenerator<Byte> {
+
+ @Override
+ protected Byte generateRandomNonNullValue(Random rand) {
+ return (byte) (rand.nextInt((Byte.MAX_VALUE - Byte.MIN_VALUE) / 2)
+ - Math.abs(Byte.MIN_VALUE / 2));
+ }
+
+ @Override
+ protected Byte[] initializeFixedPointValues() {
+ return new Byte[] {-23, -1, 17, 33};
+ }
+ }
+
+ private static class ShortBatchGenerator extends BatchGenerator<Short> {
+
+ @Override
+ protected Short generateRandomNonNullValue(Random rand) {
+ return (short) (rand.nextInt((Short.MAX_VALUE - Short.MIN_VALUE) / 2)
+ + (Short.MIN_VALUE / 2));
+ }
+
+ @Override
+ protected Short[] initializeFixedPointValues() {
+ return new Short[] {-257, -75, 197, 359};
+ }
+
+ }
+
+ private static class IntegerBatchGenerator extends BatchGenerator<Integer> {
+ @Override
+ protected Integer generateRandomNonNullValue(Random rand) {
+ return rand.nextInt(Integer.MAX_VALUE) + (Integer.MIN_VALUE / 2);
+ }
+
+ @Override
+ protected Integer[] initializeFixedPointValues() {
+ return new Integer[] {-3728, -563, 762, 6981};
+ }
+ }
+
+ private static class LongBatchGenerator extends BatchGenerator<Long> {
+
+ @Override
+ protected Long generateRandomNonNullValue(Random rand) {
+ return (long) rand.nextInt();
+ }
+
+ @Override
+ protected Long[] initializeFixedPointValues() {
+ return new Long[] {(long) -89010, (long) -6432, (long) 3569, (long) 988888};
+ }
+ }
+
+ private static class FloatBatchGenerator extends BatchGenerator<Float> {
+
+ private final ByteBatchGenerator byteGenerator = new ByteBatchGenerator();
+
+ @Override
+ protected Float generateRandomNonNullValue(Random rand) {
+ return (float) byteGenerator.generateRandomNonNullValue(rand);
+ }
+
+ @Override
+ protected Float[] initializeFixedPointValues() {
+ return new Float[] {(float) -26.28, (float) -1.389, (float) 10.175, (float) 79.553};
+ }
+
+ }
+
+ private static class DoubleBatchGenerator extends BatchGenerator<Double> {
+
+ private final ShortBatchGenerator shortGenerator = new ShortBatchGenerator();
+
+ @Override
+ protected Double generateRandomNonNullValue(Random rand) {
+ return (double) shortGenerator.generateRandomNonNullValue(rand);
+ }
+
+ @Override
+ protected Double[] initializeFixedPointValues() {
+ return new Double[] {-5638.15, -863.257, 2563.58, 9763215.5639};
+ }
+
+ }
+
+ private static class BooleanBatchGenerator extends BatchGenerator<Boolean> {
+ @Override
+ protected Boolean generateRandomNonNullValue(Random rand) {
+ return rand.nextBoolean();
+ }
+
+ @Override
+ protected Boolean[] initializeFixedPointValues() {
+ return null;
+ }
+ }
+
+ private static class StringBatchGenerator extends BatchGenerator<String> {
+
+ @Override
+ protected String generateRandomNonNullValue(Random rand) {
+ int length = rand.nextInt(20) + 5;
+ char[] values = new char[length];
+ for (int j = 0; j < length; j++) {
+ switch (rand.nextInt(3)) {
+ case 0:
+ values[j] = (char) (rand.nextInt((int) 'z' - (int) 'a') + (int) 'a');
+ break;
+ case 1:
+ values[j] = (char) (rand.nextInt((int) 'Z' - (int) 'A') + (int) 'A');
+ break;
+ case 2:
+ values[j] = (char) (rand.nextInt((int) '9' - (int) '0') + (int) '0');
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+ return new String(values);
+ }
+
+ @Override
+ protected String[] initializeFixedPointValues() {
+ return new String[] {"a", "b", "ss", "10"};
+ }
+
+ }
+
+ private static class TimestampBatchGenerator extends BatchGenerator<Timestamp> {
+
+ private final ShortBatchGenerator shortGen = new ShortBatchGenerator();
+
+ @Override
+ protected Timestamp generateRandomNonNullValue(Random rand) {
+ return new Timestamp(shortGen.generateRandomNonNullValue(rand));
+ }
+
+ @Override
+ protected Timestamp[] initializeFixedPointValues() {
+ // TODO Auto-generated method stub
+ return new Timestamp[] {
+ new Timestamp(-29071),
+ new Timestamp(-10669),
+ new Timestamp(16558),
+ new Timestamp(31808)
+ };
+ }
+ }
+
+ private static final Map<Class, BatchGenerator> TYPE_TO_BATCH_GEN_MAP;
+ static {
+ TYPE_TO_BATCH_GEN_MAP = new HashMap<Class, BatchGenerator>();
+ TYPE_TO_BATCH_GEN_MAP.put(Boolean.class, new BooleanBatchGenerator());
+
+ TYPE_TO_BATCH_GEN_MAP.put(Byte.class, new ByteBatchGenerator());
+ TYPE_TO_BATCH_GEN_MAP.put(Integer.class, new IntegerBatchGenerator());
+ TYPE_TO_BATCH_GEN_MAP.put(Long.class, new LongBatchGenerator());
+ TYPE_TO_BATCH_GEN_MAP.put(Short.class, new ShortBatchGenerator());
+
+ TYPE_TO_BATCH_GEN_MAP.put(Float.class, new FloatBatchGenerator());
+ TYPE_TO_BATCH_GEN_MAP.put(Double.class, new DoubleBatchGenerator());
+
+ TYPE_TO_BATCH_GEN_MAP.put(String.class, new StringBatchGenerator());
+
+ TYPE_TO_BATCH_GEN_MAP.put(Timestamp.class, new TimestampBatchGenerator());
+ }
+
+ /**
+ * Generates an orc file based on the provided record class in the specified file system
+ * at the output path.
+ *
+ * @param conf the configuration used to initialize the orc writer
+ * @param fs the file system to which will contain the generated orc file
+ * @param outputPath the path where the generated orc will be placed
+ * @param recordClass a class the defines the record format for the generated orc file, this
+ * class must have exactly one constructor.
+ */
+ public static void generateOrcFile(Configuration conf, FileSystem fs, Path outputPath,
+ Class recordClass)
+ throws IOException, InstantiationException,
+ IllegalAccessException, InvocationTargetException {
+
+ ObjectInspector inspector;
+ synchronized (TestVectorizedORCReader.class) {
+ inspector = ObjectInspectorFactory.getReflectionObjectInspector(
+ recordClass, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+
+ Writer writer = OrcFile.createWriter(
+ fs,
+ outputPath,
+ conf,
+ inspector,
+ 100000,
+ CompressionKind.ZLIB,
+ 10000,
+ 10000);
+
+ try {
+ Constructor[] constructors = recordClass.getConstructors();
+
+ if (constructors.length != 1) {
+ throw new UnsupportedOperationException(
+ "The provided recordClass must have exactly one constructor.");
+ }
+
+ BatchDataDistribution[] dataDist = BatchDataDistribution.values();
+ Class[] columns = constructors[0].getParameterTypes();
+ for (int i = 0; i < dataDist.length * 3; i++) {
+ Object[][] rows = new Object[columns.length][VectorizedRowBatch.DEFAULT_SIZE];
+ for (int c = 0; c < columns.length; c++) {
+ if (!TYPE_TO_BATCH_GEN_MAP.containsKey(columns[c])) {
+ throw new UnsupportedOperationException("No batch generator defined for type "
+ + columns[c].getName());
+ }
+ rows[c] = TYPE_TO_BATCH_GEN_MAP.get(
+ columns[c]).generateBatch(dataDist[(i + c) % dataDist.length]);
+ }
+
+ for (int r = 0; r < VectorizedRowBatch.DEFAULT_SIZE; r++) {
+ Object[] row = new Object[columns.length];
+ for (int c = 0; c < columns.length; c++) {
+ row[c] = rows[c][r];
+ }
+ writer.addRow(
+ constructors[0].newInstance(row));
+ }
+ }
+ } finally {
+ writer.close();
+ }
+ }
+}
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_0.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_0.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_0.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_0.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,27 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT AVG(cbigint),
+ (-(AVG(cbigint))),
+ (-6432 + AVG(cbigint)),
+ STDDEV_POP(cbigint),
+ (-((-6432 + AVG(cbigint)))),
+ ((-((-6432 + AVG(cbigint)))) + (-6432 + AVG(cbigint))),
+ VAR_SAMP(cbigint),
+ (-((-6432 + AVG(cbigint)))),
+ (-6432 + (-((-6432 + AVG(cbigint))))),
+ (-((-6432 + AVG(cbigint)))),
+ ((-((-6432 + AVG(cbigint)))) / (-((-6432 + AVG(cbigint))))),
+ COUNT(*),
+ SUM(cfloat),
+ (VAR_SAMP(cbigint) % STDDEV_POP(cbigint)),
+ (-(VAR_SAMP(cbigint))),
+ ((-((-6432 + AVG(cbigint)))) * (-(AVG(cbigint)))),
+ MIN(ctinyint),
+ (-(MIN(ctinyint)))
+FROM alltypesorc
+WHERE (((cstring2 LIKE '%b%')
+ OR ((79.553 != cint)
+ OR (cbigint < cdouble)))
+ OR ((ctinyint >= csmallint)
+ AND ((cboolean2 = 1)
+ AND (3569 = ctinyint))));
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_1.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_1.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_1.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_1.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,21 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT VAR_POP(ctinyint),
+ (VAR_POP(ctinyint) / -26.28),
+ SUM(cfloat),
+ (-1.389 + SUM(cfloat)),
+ (SUM(cfloat) * (-1.389 + SUM(cfloat))),
+ MAX(ctinyint),
+ (-((SUM(cfloat) * (-1.389 + SUM(cfloat))))),
+ MAX(cint),
+ (MAX(cint) * 79.553),
+ VAR_SAMP(cdouble),
+ (10.175 % (-((SUM(cfloat) * (-1.389 + SUM(cfloat)))))),
+ COUNT(cint),
+ (-563 % MAX(cint))
+FROM alltypesorc
+WHERE (((cdouble > ctinyint)
+ AND (cboolean2 > 0))
+ OR ((cbigint < ctinyint)
+ OR ((cint > cbigint)
+ OR (cboolean1 < 0))));
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_10.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_10.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_10.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_10.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,24 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT cdouble,
+ ctimestamp1,
+ ctinyint,
+ cboolean1,
+ cstring1,
+ (-(cdouble)),
+ (cdouble + csmallint),
+ ((cdouble + csmallint) % 33),
+ (-(cdouble)),
+ (ctinyint % cdouble),
+ (ctinyint % csmallint),
+ (-(cdouble)),
+ (cbigint * (ctinyint % csmallint)),
+ (9763215.5639 - (cdouble + csmallint)),
+ (-((-(cdouble))))
+FROM alltypesorc
+WHERE (((cstring2 <= '10')
+ OR ((ctinyint > cdouble)
+ AND (-5638.15 >= ctinyint)))
+ OR ((cdouble > 6981)
+ AND ((csmallint = 9763215.5639)
+ OR (cstring1 LIKE '%a'))));
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_11.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_11.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_11.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_11.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,15 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT cstring1,
+ cboolean1,
+ cdouble,
+ ctimestamp1,
+ (-3728 * csmallint),
+ (cdouble - 9763215.5639),
+ (-(cdouble)),
+ ((-(cdouble)) + 6981),
+ (cdouble * -5638.15)
+FROM alltypesorc
+WHERE ((cstring2 = cstring1)
+ OR ((ctimestamp1 IS NULL)
+ AND (cstring1 LIKE '%a')));
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_12.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_12.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_12.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_12.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,32 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT cbigint,
+ cboolean1,
+ cstring1,
+ ctimestamp1,
+ cdouble,
+ (-6432 * cdouble),
+ (-(cbigint)),
+ COUNT(cbigint),
+ (cbigint * COUNT(cbigint)),
+ STDDEV_SAMP(cbigint),
+ ((-6432 * cdouble) / -6432),
+ (-(((-6432 * cdouble) / -6432))),
+ AVG(cdouble),
+ (-((-6432 * cdouble))),
+ (-5638.15 + cbigint),
+ SUM(cbigint),
+ (AVG(cdouble) / (-6432 * cdouble)),
+ AVG(cdouble),
+ (-((-(((-6432 * cdouble) / -6432))))),
+ (((-6432 * cdouble) / -6432) + (-((-6432 * cdouble)))),
+ STDDEV_POP(cdouble)
+FROM alltypesorc
+WHERE (((ctimestamp1 IS NULL)
+ AND ((cboolean1 >= cboolean2)
+ OR (ctinyint != csmallint)))
+ AND ((cstring1 LIKE '%a')
+ OR ((cboolean2 <= 1)
+ AND (cbigint >= csmallint))))
+GROUP BY cbigint, cboolean1, cstring1, ctimestamp1, cdouble
+ORDER BY ctimestamp1, cdouble;
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_13.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_13.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_13.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_13.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,31 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT cboolean1,
+ ctinyint,
+ ctimestamp1,
+ cfloat,
+ cstring1,
+ (-(ctinyint)),
+ MAX(ctinyint),
+ ((-(ctinyint)) + MAX(ctinyint)),
+ SUM(cfloat),
+ (SUM(cfloat) * ((-(ctinyint)) + MAX(ctinyint))),
+ (-(SUM(cfloat))),
+ (79.553 * cfloat),
+ STDDEV_POP(cfloat),
+ (-(SUM(cfloat))),
+ STDDEV_POP(ctinyint),
+ (((-(ctinyint)) + MAX(ctinyint)) - 10.175),
+ (-((-(SUM(cfloat))))),
+ (-26.28 / (-((-(SUM(cfloat)))))),
+ MAX(cfloat),
+ ((SUM(cfloat) * ((-(ctinyint)) + MAX(ctinyint))) / ctinyint),
+ MIN(ctinyint)
+FROM alltypesorc
+WHERE (((cfloat < 3569)
+ AND ((10.175 >= cdouble)
+ AND (cboolean1 != 1)))
+ OR ((ctimestamp1 > -29071)
+ AND ((ctimestamp2 != -29071)
+ AND (ctinyint < 9763215.5639))))
+GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1;
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_14.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_14.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_14.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_14.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,33 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT ctimestamp1,
+ cfloat,
+ cstring1,
+ cboolean1,
+ cdouble,
+ (-26.28 + cdouble),
+ (-((-26.28 + cdouble))),
+ STDDEV_SAMP((-((-26.28 + cdouble)))),
+ (cfloat * -26.28),
+ MAX(cfloat),
+ (-(cfloat)),
+ (-(MAX(cfloat))),
+ ((-((-26.28 + cdouble))) / 10.175),
+ STDDEV_POP(cfloat),
+ COUNT(cfloat),
+ (-(((-((-26.28 + cdouble))) / 10.175))),
+ (-1.389 % STDDEV_SAMP((-((-26.28 + cdouble))))),
+ (cfloat - cdouble),
+ VAR_POP(cfloat),
+ (VAR_POP(cfloat) % 10.175),
+ VAR_SAMP(cfloat),
+ (-((cfloat - cdouble)))
+FROM alltypesorc
+WHERE (((ctinyint <= cbigint)
+ AND ((cint <= cdouble)
+ OR (ctimestamp2 < ctimestamp1)))
+ AND ((cdouble < ctinyint)
+ AND ((cbigint > -257)
+ OR (cfloat < cint))))
+GROUP BY ctimestamp1, cfloat, cstring1, cboolean1, cdouble
+ORDER BY cstring1, cfloat, cdouble;
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_15.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_15.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_15.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_15.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,30 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT cfloat,
+ cboolean1,
+ cdouble,
+ cstring1,
+ ctinyint,
+ cint,
+ ctimestamp1,
+ STDDEV_SAMP(cfloat),
+ (-26.28 - cint),
+ MIN(cdouble),
+ (cdouble * 79.553),
+ (33 % cfloat),
+ STDDEV_SAMP(ctinyint),
+ VAR_POP(ctinyint),
+ (-23 % cdouble),
+ (-(ctinyint)),
+ VAR_SAMP(cint),
+ (cint - cfloat),
+ (-23 % ctinyint),
+ (-((-26.28 - cint))),
+ STDDEV_POP(cint)
+FROM alltypesorc
+WHERE (((cstring2 LIKE '%ss%')
+ OR (cstring1 LIKE '10%'))
+ OR ((cint >= -75)
+ AND ((ctinyint = csmallint)
+ AND (cdouble >= -3728))))
+GROUP BY cfloat, cboolean1, cdouble, cstring1, ctinyint, cint, ctimestamp1;
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_16.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_16.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_16.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_16.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,20 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT cstring1,
+ cdouble,
+ ctimestamp1,
+ (cdouble - 9763215.5639),
+ (-((cdouble - 9763215.5639))),
+ COUNT(cdouble),
+ STDDEV_SAMP(cdouble),
+ (-(STDDEV_SAMP(cdouble))),
+ (STDDEV_SAMP(cdouble) * COUNT(cdouble)),
+ MIN(cdouble),
+ (9763215.5639 / cdouble),
+ (COUNT(cdouble) / -1.389),
+ STDDEV_SAMP(cdouble)
+FROM alltypesorc
+WHERE ((cstring2 LIKE '%b%')
+ AND ((cdouble >= -1.389)
+ OR (cstring1 < 'a')))
+GROUP BY cstring1, cdouble, ctimestamp1;
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_2.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_2.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_2.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_2.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,23 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT AVG(csmallint),
+ (AVG(csmallint) % -563),
+ (AVG(csmallint) + 762),
+ SUM(cfloat),
+ VAR_POP(cbigint),
+ (-(VAR_POP(cbigint))),
+ (SUM(cfloat) - AVG(csmallint)),
+ COUNT(*),
+ (-((SUM(cfloat) - AVG(csmallint)))),
+ (VAR_POP(cbigint) - 762),
+ MIN(ctinyint),
+ ((-(VAR_POP(cbigint))) + MIN(ctinyint)),
+ AVG(cdouble),
+ (((-(VAR_POP(cbigint))) + MIN(ctinyint)) - SUM(cfloat))
+FROM alltypesorc
+WHERE (((ctimestamp1 < ctimestamp2)
+ AND ((cstring2 LIKE 'b%')
+ AND (cfloat <= -5638.15)))
+ OR ((cdouble < ctinyint)
+ AND ((-10669 != ctimestamp2)
+ OR (359 > cint))));
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_3.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_3.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_3.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_3.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,25 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT STDDEV_SAMP(csmallint),
+ (STDDEV_SAMP(csmallint) - 10.175),
+ STDDEV_POP(ctinyint),
+ (STDDEV_SAMP(csmallint) * (STDDEV_SAMP(csmallint) - 10.175)),
+ (-(STDDEV_POP(ctinyint))),
+ (STDDEV_SAMP(csmallint) % 79.553),
+ (-((STDDEV_SAMP(csmallint) * (STDDEV_SAMP(csmallint) - 10.175)))),
+ STDDEV_SAMP(cfloat),
+ (-(STDDEV_SAMP(csmallint))),
+ SUM(cfloat),
+ ((-((STDDEV_SAMP(csmallint) * (STDDEV_SAMP(csmallint) - 10.175)))) / (STDDEV_SAMP(csmallint) - 10.175)),
+ (-((STDDEV_SAMP(csmallint) - 10.175))),
+ AVG(cint),
+ (-3728 - STDDEV_SAMP(csmallint)),
+ STDDEV_POP(cint),
+ (AVG(cint) / STDDEV_SAMP(cfloat))
+FROM alltypesorc
+WHERE (((cint <= cfloat)
+ AND ((79.553 != cbigint)
+ AND (ctimestamp2 = -29071)))
+ OR ((cbigint > cdouble)
+ AND ((79.553 <= csmallint)
+ AND (ctimestamp1 > ctimestamp2))));
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_4.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_4.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_4.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_4.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,23 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT SUM(cint),
+ (SUM(cint) * -563),
+ (-3728 + SUM(cint)),
+ STDDEV_POP(cdouble),
+ (-(STDDEV_POP(cdouble))),
+ AVG(cdouble),
+ ((SUM(cint) * -563) % SUM(cint)),
+ (((SUM(cint) * -563) % SUM(cint)) / AVG(cdouble)),
+ VAR_POP(cdouble),
+ (-((((SUM(cint) * -563) % SUM(cint)) / AVG(cdouble)))),
+ ((-3728 + SUM(cint)) - (SUM(cint) * -563)),
+ MIN(ctinyint),
+ MIN(ctinyint),
+ (MIN(ctinyint) * (-((((SUM(cint) * -563) % SUM(cint)) / AVG(cdouble)))))
+FROM alltypesorc
+WHERE (((csmallint >= cint)
+ OR ((-89010 >= ctinyint)
+ AND (cdouble > 79.553)))
+ OR ((-563 != cbigint)
+ AND ((ctinyint != cbigint)
+ OR (-3728 >= cdouble))));
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_5.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_5.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_5.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_5.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,20 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT MAX(csmallint),
+ (MAX(csmallint) * -75),
+ COUNT(*),
+ ((MAX(csmallint) * -75) / COUNT(*)),
+ (6981 * MAX(csmallint)),
+ MIN(csmallint),
+ (-(MIN(csmallint))),
+ (197 % ((MAX(csmallint) * -75) / COUNT(*))),
+ SUM(cint),
+ MAX(ctinyint),
+ (-(MAX(ctinyint))),
+ ((-(MAX(ctinyint))) + MAX(ctinyint))
+FROM alltypesorc
+WHERE (((cboolean2 IS NOT NULL)
+ AND (cstring1 LIKE '%b%'))
+ OR ((ctinyint = cdouble)
+ AND ((ctimestamp2 IS NOT NULL)
+ AND (cstring2 LIKE 'a'))));
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_6.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_6.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_6.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_6.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,21 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT cboolean1,
+ cfloat,
+ cstring1,
+ (988888 * csmallint),
+ (-(csmallint)),
+ (-(cfloat)),
+ (-26.28 / cfloat),
+ (cfloat * 359),
+ (cint % ctinyint),
+ (-(cdouble)),
+ (ctinyint - -75),
+ (762 * (cint % ctinyint))
+FROM alltypesorc
+WHERE ((ctinyint != 0)
+ AND ((((cboolean1 <= 0)
+ AND (cboolean2 >= cboolean1))
+ OR ((cbigint IS NOT NULL)
+ AND ((cstring2 LIKE '%a')
+ OR (cfloat <= -257))))));
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_7.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_7.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_7.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_7.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,25 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT cboolean1,
+ cbigint,
+ csmallint,
+ ctinyint,
+ ctimestamp1,
+ cstring1,
+ (cbigint + cbigint),
+ (csmallint % -257),
+ (-(csmallint)),
+ (-(ctinyint)),
+ ((-(ctinyint)) + 17),
+ (cbigint * (-(csmallint))),
+ (cint % csmallint),
+ (-(ctinyint)),
+ ((-(ctinyint)) % ctinyint)
+FROM alltypesorc
+WHERE ((ctinyint != 0)
+ AND (((ctimestamp1 <= 0)
+ OR ((ctinyint = cint)
+ OR (cstring2 LIKE 'ss')))
+ AND ((988888 < cdouble)
+ OR ((ctimestamp2 > -29071)
+ AND (3569 >= cdouble)))));
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_8.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_8.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_8.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_8.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,23 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT ctimestamp1,
+ cdouble,
+ cboolean1,
+ cstring1,
+ cfloat,
+ (-(cdouble)),
+ (-5638.15 - cdouble),
+ (cdouble * -257),
+ (cint + cfloat),
+ ((-(cdouble)) + cbigint),
+ (-(cdouble)),
+ (-1.389 - cfloat),
+ (-(cfloat)),
+ ((-5638.15 - cdouble) + (cint + cfloat))
+FROM alltypesorc
+WHERE (((cstring2 IS NOT NULL)
+ AND ((ctimestamp1 <= -29071)
+ AND (ctimestamp2 != 16558)))
+ OR ((cfloat < -6432)
+ OR ((cboolean1 IS NOT NULL)
+ AND (cdouble = 988888))));
+
Added: hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_9.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_9.q?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_9.q (added)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/vectorization_9.q Mon Aug 5 21:06:24 2013
@@ -0,0 +1,24 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT cfloat,
+ cstring1,
+ cint,
+ ctimestamp1,
+ cdouble,
+ cbigint,
+ (cfloat / ctinyint),
+ (cint % cbigint),
+ (-(cdouble)),
+ (cdouble + (cfloat / ctinyint)),
+ (cdouble / cint),
+ (-((-(cdouble)))),
+ (9763215.5639 % cbigint),
+ (2563.58 + (-((-(cdouble)))))
+FROM alltypesorc
+WHERE (((cbigint > -23)
+ AND ((cdouble != 988888)
+ OR (cint > -863.257)))
+ AND ((ctinyint >= 33)
+ OR ((csmallint >= cbigint)
+ OR (cfloat = cdouble))))
+ORDER BY cbigint, cfloat;
+
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/add_part_exist.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/add_part_exist.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/add_part_exist.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/add_part_exist.q.out Mon Aug 5 21:06:24 2013
@@ -75,6 +75,7 @@ PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
+alltypesorc
src
src1
src_json
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/alter1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/alter1.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/alter1.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/alter1.q.out Mon Aug 5 21:06:24 2013
@@ -168,6 +168,7 @@ PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
+alltypesorc
src
src1
src_json
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/alter2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/alter2.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/alter2.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/alter2.q.out Mon Aug 5 21:06:24 2013
@@ -172,6 +172,7 @@ PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
+alltypesorc
src
src1
src_json
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/alter3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/alter3.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/alter3.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/alter3.q.out Mon Aug 5 21:06:24 2013
@@ -184,6 +184,7 @@ POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
POSTHOOK: Lineage: alter3 PARTITION(pcol1=test_part:,pcol2=test_part:).col1 SIMPLE [(alter3_src)alter3_src.FieldSchema(name:col1, type:string, comment:null), ]
POSTHOOK: Lineage: alter3_like PARTITION(pcol1=test_part:,pcol2=test_part:).col1 SIMPLE [(alter3_src)alter3_src.FieldSchema(name:col1, type:string, comment:null), ]
+alltypesorc
src
src1
src_json
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/alter4.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/alter4.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/alter4.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/alter4.q.out Mon Aug 5 21:06:24 2013
@@ -41,6 +41,7 @@ PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
+alltypesorc
src
src1
src_json
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/alter5.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/alter5.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/alter5.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/alter5.q.out Mon Aug 5 21:06:24 2013
@@ -115,6 +115,7 @@ PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
POSTHOOK: Lineage: alter5 PARTITION(dt=a).col1 SIMPLE [(alter5_src)alter5_src.FieldSchema(name:col1, type:string, comment:null), ]
+alltypesorc
src
src1
src_json
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/alter_index.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/alter_index.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/alter_index.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/alter_index.q.out Mon Aug 5 21:06:24 2013
@@ -37,6 +37,7 @@ PREHOOK: query: show tables
PREHOOK: type: SHOWTABLES
POSTHOOK: query: show tables
POSTHOOK: type: SHOWTABLES
+alltypesorc
src
src1
src_json
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/alter_rename_partition.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/alter_rename_partition.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/alter_rename_partition.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/alter_rename_partition.q.out Mon Aug 5 21:06:24 2013
@@ -12,6 +12,7 @@ PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
+alltypesorc
src
src1
src_json
@@ -130,6 +131,7 @@ PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
POSTHOOK: Lineage: alter_rename_partition PARTITION(pcol1=old_part1:,pcol2=old_part2:).col1 SIMPLE [(alter_rename_partition_src)alter_rename_partition_src.FieldSchema(name:col1, type:string, comment:null), ]
+alltypesorc
src
src1
src_json
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/describe_table_json.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/describe_table_json.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/describe_table_json.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/describe_table_json.q.out Mon Aug 5 21:06:24 2013
@@ -7,7 +7,7 @@ PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
-{"tables":["jsontable","src","src1","src_json","src_sequencefile","src_thrift","srcbucket","srcbucket2","srcpart"]}
+{"tables":["alltypesorc","jsontable","src","src1","src_json","src_sequencefile","src_thrift","srcbucket","srcbucket2","srcpart"]}
PREHOOK: query: SHOW TABLES LIKE 'json*'
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES LIKE 'json*'
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/index_creation.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/index_creation.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/index_creation.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/index_creation.q.out Mon Aug 5 21:06:24 2013
@@ -306,6 +306,7 @@ POSTHOOK: Lineage: default___t_x2__._off
POSTHOOK: Lineage: default___t_x__._bucketname SIMPLE [(_t)_t.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default___t_x__._j SIMPLE [(_t)_t.FieldSchema(name:_j, type:int, comment:null), ]
POSTHOOK: Lineage: default___t_x__._offsets EXPRESSION [(_t)_t.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
+alltypesorc
src
src1
src_json
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/input2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/input2.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/input2.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/input2.q.out Mon Aug 5 21:06:24 2013
@@ -31,6 +31,7 @@ PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
+alltypesorc
src
src1
src_json
@@ -53,6 +54,7 @@ PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
+alltypesorc
src
src1
src_json
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/input3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/input3.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/input3.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/input3.q.out Mon Aug 5 21:06:24 2013
@@ -25,6 +25,7 @@ PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
+alltypesorc
src
src1
src_json
@@ -114,6 +115,7 @@ PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
+alltypesorc
src
src1
src_json
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/rename_column.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/rename_column.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/rename_column.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/rename_column.q.out Mon Aug 5 21:06:24 2013
@@ -127,6 +127,7 @@ PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
+alltypesorc
src
src1
src_json
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/show_tables.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/show_tables.q.out?rev=1510732&r1=1510731&r2=1510732&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/show_tables.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/show_tables.q.out Mon Aug 5 21:06:24 2013
@@ -112,6 +112,7 @@ PREHOOK: query: SHOW TABLES FROM default
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES FROM default
POSTHOOK: type: SHOWTABLES
+alltypesorc
shtb_test1
shtb_test2
src
@@ -133,6 +134,7 @@ PREHOOK: query: SHOW TABLES IN default
PREHOOK: type: SHOWTABLES
POSTHOOK: query: SHOW TABLES IN default
POSTHOOK: type: SHOWTABLES
+alltypesorc
shtb_test1
shtb_test2
src
Added: hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_0.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_0.q.out?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_0.q.out (added)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_0.q.out Mon Aug 5 21:06:24 2013
@@ -0,0 +1,58 @@
+WARNING: Comparing a bigint and a double may result in a loss of precision.
+PREHOOK: query: SELECT AVG(cbigint),
+ (-(AVG(cbigint))),
+ (-6432 + AVG(cbigint)),
+ STDDEV_POP(cbigint),
+ (-((-6432 + AVG(cbigint)))),
+ ((-((-6432 + AVG(cbigint)))) + (-6432 + AVG(cbigint))),
+ VAR_SAMP(cbigint),
+ (-((-6432 + AVG(cbigint)))),
+ (-6432 + (-((-6432 + AVG(cbigint))))),
+ (-((-6432 + AVG(cbigint)))),
+ ((-((-6432 + AVG(cbigint)))) / (-((-6432 + AVG(cbigint))))),
+ COUNT(*),
+ SUM(cfloat),
+ (VAR_SAMP(cbigint) % STDDEV_POP(cbigint)),
+ (-(VAR_SAMP(cbigint))),
+ ((-((-6432 + AVG(cbigint)))) * (-(AVG(cbigint)))),
+ MIN(ctinyint),
+ (-(MIN(ctinyint)))
+FROM alltypesorc
+WHERE (((cstring2 LIKE '%b%')
+ OR ((79.553 != cint)
+ OR (cbigint < cdouble)))
+ OR ((ctinyint >= csmallint)
+ AND ((cboolean2 = 1)
+ AND (3569 = ctinyint))))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT AVG(cbigint),
+ (-(AVG(cbigint))),
+ (-6432 + AVG(cbigint)),
+ STDDEV_POP(cbigint),
+ (-((-6432 + AVG(cbigint)))),
+ ((-((-6432 + AVG(cbigint)))) + (-6432 + AVG(cbigint))),
+ VAR_SAMP(cbigint),
+ (-((-6432 + AVG(cbigint)))),
+ (-6432 + (-((-6432 + AVG(cbigint))))),
+ (-((-6432 + AVG(cbigint)))),
+ ((-((-6432 + AVG(cbigint)))) / (-((-6432 + AVG(cbigint))))),
+ COUNT(*),
+ SUM(cfloat),
+ (VAR_SAMP(cbigint) % STDDEV_POP(cbigint)),
+ (-(VAR_SAMP(cbigint))),
+ ((-((-6432 + AVG(cbigint)))) * (-(AVG(cbigint)))),
+ MIN(ctinyint),
+ (-(MIN(ctinyint)))
+FROM alltypesorc
+WHERE (((cstring2 LIKE '%b%')
+ OR ((79.553 != cint)
+ OR (cbigint < cdouble)))
+ OR ((ctinyint >= csmallint)
+ AND ((cboolean2 = 1)
+ AND (3569 = ctinyint))))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+-3.875652215945533E8 3.875652215945533E8 -3.875716535945533E8 1.436387455459401E9 3.875716535945533E8 0.0 2.06347151720204902E18 3.875716535945533E8 3.875652215945533E8 3.875716535945533E8 1.0 10934 -37224.52399241924 1.0517370547117279E9 -2.06347151720204902E18 1.5020929380914048E17 -64 64
Added: hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_1.q.out?rev=1510732&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_1.q.out (added)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/vectorization_1.q.out Mon Aug 5 21:06:24 2013
@@ -0,0 +1,45 @@
+PREHOOK: query: SELECT VAR_POP(ctinyint),
+ (VAR_POP(ctinyint) / -26.28),
+ SUM(cfloat),
+ (-1.389 + SUM(cfloat)),
+ (SUM(cfloat) * (-1.389 + SUM(cfloat))),
+ MAX(ctinyint),
+ (-((SUM(cfloat) * (-1.389 + SUM(cfloat))))),
+ MAX(cint),
+ (MAX(cint) * 79.553),
+ VAR_SAMP(cdouble),
+ (10.175 % (-((SUM(cfloat) * (-1.389 + SUM(cfloat)))))),
+ COUNT(cint),
+ (-563 % MAX(cint))
+FROM alltypesorc
+WHERE (((cdouble > ctinyint)
+ AND (cboolean2 > 0))
+ OR ((cbigint < ctinyint)
+ OR ((cint > cbigint)
+ OR (cboolean1 < 0))))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT VAR_POP(ctinyint),
+ (VAR_POP(ctinyint) / -26.28),
+ SUM(cfloat),
+ (-1.389 + SUM(cfloat)),
+ (SUM(cfloat) * (-1.389 + SUM(cfloat))),
+ MAX(ctinyint),
+ (-((SUM(cfloat) * (-1.389 + SUM(cfloat))))),
+ MAX(cint),
+ (MAX(cint) * 79.553),
+ VAR_SAMP(cdouble),
+ (10.175 % (-((SUM(cfloat) * (-1.389 + SUM(cfloat)))))),
+ COUNT(cint),
+ (-563 % MAX(cint))
+FROM alltypesorc
+WHERE (((cdouble > ctinyint)
+ AND (cboolean2 > 0))
+ OR ((cbigint < ctinyint)
+ OR ((cint > cbigint)
+ OR (cboolean1 < 0))))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+1074.8302575472298 -40.89917266161453 -21997.674998402596 -21999.063998402595 4.839282601059194E8 62 -4.839282601059194E8 1073680599 8.5414512692247E10 7.569848642620903E10 10.175 3745 -563