You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/08/26 23:42:21 UTC
svn commit: r1517707 [11/17] - in /hive/branches/tez: ./
beeline/src/java/org/apache/hive/beeline/ bin/ bin/ext/
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/conf/ conf/
contrib/src/java/org/apache/hadoop/hive/c...
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java Mon Aug 26 21:42:12 2013
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import java.util.ArrayList;
+
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -26,13 +28,11 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import java.util.ArrayList;
-
@Description(name ="inline", value= "_FUNC_( ARRAY( STRUCT()[,STRUCT()] "
+ "- explodes and array and struct into a table")
public class GenericUDTFInline extends GenericUDTF {
- private ListObjectInspector li;
+ private transient ListObjectInspector li;
public GenericUDTFInline(){
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java Mon Aug 26 21:42:12 2013
@@ -66,7 +66,7 @@ public class GenericUDTFJSONTuple extend
Text[] retCols; // array of returned column values
Text[] cols; // object pool of non-null Text, avoid creating objects all the time
Object[] nullCols; // array of null column values
- ObjectInspector[] inputOIs; // input ObjectInspectors
+ private transient ObjectInspector[] inputOIs; // input ObjectInspectors
boolean pathParsed = false;
boolean seenErrors = false;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java Mon Aug 26 21:42:12 2013
@@ -18,16 +18,15 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import java.util.List;
import java.util.ArrayList;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils.ReturnObjectInspectorResolver;
-import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantIntObjectInspector;
import org.apache.hadoop.io.IntWritable;
@@ -48,9 +47,9 @@ public class GenericUDTFStack extends Ge
public void close() throws HiveException {
}
- ArrayList<ObjectInspector> argOIs = new ArrayList<ObjectInspector>();
+ private transient List<ObjectInspector> argOIs = new ArrayList<ObjectInspector>();
Object[] forwardObj = null;
- ArrayList<ReturnObjectInspectorResolver> returnOIResolvers =
+ private transient ArrayList<ReturnObjectInspectorResolver> returnOIResolvers =
new ArrayList<ReturnObjectInspectorResolver>();
IntWritable numRows = null;
Integer numCols = null;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NGramEstimator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NGramEstimator.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NGramEstimator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NGramEstimator.java Mon Aug 26 21:42:12 2013
@@ -104,7 +104,19 @@ public class NGramEstimator {
Collections.sort(list, new Comparator<Map.Entry<ArrayList<String>, Double>>() {
public int compare(Map.Entry<ArrayList<String>, Double> o1,
Map.Entry<ArrayList<String>, Double> o2) {
- return o2.getValue().compareTo(o1.getValue());
+ int result = o2.getValue().compareTo(o1.getValue());
+ if (result != 0)
+ return result;
+
+ ArrayList<String> key1 = o1.getKey();
+ ArrayList<String> key2 = o2.getKey();
+ for (int i = 0; i < key1.size() && i < key2.size(); i++) {
+ result = key1.get(i).compareTo(key2.get(i));
+ if (result != 0)
+ return result;
+ }
+
+ return key1.size() - key2.size();
}
});
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java Mon Aug 26 21:42:12 2013
@@ -891,7 +891,7 @@ public class NPath extends TableFunction
}
public static Object getSelectListInput(Object currRow, ObjectInspector rowOI,
- PTFPartitionIterator<Object> pItr, int sz) {
+ PTFPartitionIterator<Object> pItr, int sz) throws HiveException {
ArrayList<Object> oRow = new ArrayList<Object>();
List<?> currRowAsStdObject = (List<?>) ObjectInspectorUtils
.copyToStandardObject(currRow, rowOI);
@@ -901,7 +901,7 @@ public class NPath extends TableFunction
}
public static ArrayList<Object> getPath(Object currRow, ObjectInspector rowOI,
- PTFPartitionIterator<Object> pItr, int sz) {
+ PTFPartitionIterator<Object> pItr, int sz) throws HiveException {
int idx = pItr.getIndex() - 1;
ArrayList<Object> path = new ArrayList<Object>();
path.add(ObjectInspectorUtils.copyToStandardObject(currRow, rowOI));
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java Mon Aug 26 21:42:12 2013
@@ -61,8 +61,6 @@ public abstract class TableFunctionEvalu
transient protected StructObjectInspector rawInputOI;
protected PartitionedTableFunctionDef tDef;
protected PTFDesc ptfDesc;
- String partitionClass;
- int partitionMemSize;
boolean transformsRawInput;
transient protected PTFPartition outputPartition;
@@ -100,26 +98,6 @@ public abstract class TableFunctionEvalu
this.ptfDesc = ptfDesc;
}
- public String getPartitionClass()
- {
- return partitionClass;
- }
-
- public void setPartitionClass(String partitionClass)
- {
- this.partitionClass = partitionClass;
- }
-
- public int getPartitionMemSize()
- {
- return partitionMemSize;
- }
-
- public void setPartitionMemSize(int partitionMemSize)
- {
- this.partitionMemSize = partitionMemSize;
- }
-
public StructObjectInspector getRawInputOI()
{
return rawInputOI;
@@ -145,8 +123,9 @@ public abstract class TableFunctionEvalu
PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, pItr);
if ( outputPartition == null ) {
- outputPartition = new PTFPartition(getPartitionClass(),
- getPartitionMemSize(), tDef.getOutputShape().getSerde(), OI);
+ outputPartition = PTFPartition.create(ptfDesc.getCfg(),
+ tDef.getOutputShape().getSerde(),
+ OI, tDef.getOutputShape().getOI());
}
else {
outputPartition.reset();
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java Mon Aug 26 21:42:12 2013
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.udf.pt
import java.util.ArrayList;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -69,16 +68,11 @@ public abstract class TableFunctionResol
throws SemanticException
{
this.ptfDesc = ptfDesc;
- String partitionClass = HiveConf.getVar(cfg, ConfVars.HIVE_PTF_PARTITION_PERSISTENCE_CLASS);
- int partitionMemSize = HiveConf.getIntVar(cfg, ConfVars.HIVE_PTF_PARTITION_PERSISTENT_SIZE);
evaluator = createEvaluator(ptfDesc, tDef);
evaluator.setTransformsRawInput(transformsRawInput());
evaluator.setTableDef(tDef);
evaluator.setQueryDef(ptfDesc);
- evaluator.setPartitionClass(partitionClass);
- evaluator.setPartitionMemSize(partitionMemSize);
-
}
/*
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java Mon Aug 26 21:42:12 2013
@@ -40,7 +40,6 @@ import org.apache.hadoop.hive.ql.plan.PT
import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowTableFunctionDef;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer;
-import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
@@ -59,8 +58,9 @@ public class WindowingTableFunction exte
PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, pItr);
if ( outputPartition == null ) {
- outputPartition = new PTFPartition(getPartitionClass(),
- getPartitionMemSize(), wFnDef.getOutputFromWdwFnProcessing().getSerde(), OI);
+ outputPartition = PTFPartition.create(ptfDesc.getCfg(),
+ wFnDef.getOutputFromWdwFnProcessing().getSerde(),
+ OI, wFnDef.getOutputFromWdwFnProcessing().getOI());
}
else {
outputPartition.reset();
@@ -77,11 +77,7 @@ public class WindowingTableFunction exte
ArrayList<List<?>> oColumns = new ArrayList<List<?>>();
PTFPartition iPart = pItr.getPartition();
StructObjectInspector inputOI;
- try {
- inputOI = (StructObjectInspector) iPart.getSerDe().getObjectInspector();
- } catch (SerDeException se) {
- throw new HiveException(se);
- }
+ inputOI = (StructObjectInspector) iPart.getOutputOI();
WindowTableFunctionDef wTFnDef = (WindowTableFunctionDef) getTableDef();
Order order = wTFnDef.getOrder().getExpressions().get(0).getOrder();
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/xml/GenericUDFXPath.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/xml/GenericUDFXPath.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/xml/GenericUDFXPath.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/xml/GenericUDFXPath.java Mon Aug 26 21:42:12 2013
@@ -29,8 +29,8 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.io.Text;
import org.w3c.dom.NodeList;
@@ -50,11 +50,11 @@ public class GenericUDFXPath extends Gen
private static final List<Text> emptyResult = Collections.<Text>emptyList();
- private final UDFXPathUtil xpath = new UDFXPathUtil();
+ private transient final UDFXPathUtil xpath = new UDFXPathUtil();
private final List<Text> result = new ArrayList<Text>(10);
- private Converter converterArg0;
- private Converter converterArg1;
+ private transient Converter converterArg0;
+ private transient Converter converterArg1;
@Override
public ObjectInspector initialize(ObjectInspector[] arguments)
Modified: hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto (original)
+++ hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto Mon Aug 26 21:42:12 2013
@@ -73,6 +73,8 @@ message ColumnEncoding {
enum Kind {
DIRECT = 0;
DICTIONARY = 1;
+ DIRECT_V2 = 2;
+ DICTIONARY_V2 = 3;
}
required Kind kind = 1;
optional uint32 dictionarySize = 2;
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Mon Aug 26 21:42:12 2013
@@ -891,6 +891,8 @@ public class QTestUtil {
"<string>[0-9]{10}</string>",
"<string>/.*/warehouse/.*</string>"
};
+
+ fixXml4JDK7(outf.getPath());
maskPatterns(patterns, outf.getPath());
int exitVal = executeDiffCommand(outf.getPath(), planFile, true, false);
@@ -941,6 +943,101 @@ public class QTestUtil {
return ret;
}
+ /**
+ * Fix the XML generated by JDK7 which is slightly different from what's generated by JDK6,
+ * causing 40+ test failures. There are mainly two problems:
+ *
+ * 1. object element's properties, id and class, are in reverse order, i.e.
+ * <object class="org.apache.hadoop.hive.ql.exec.MapRedTask" id="MapRedTask0">
+ * which needs to be fixed to
+ * <object id="MapRedTask0" class="org.apache.hadoop.hive.ql.exec.MapRedTask">
+ * 2. JDK introduces Enum as class, i.e.
+ * <object id="GenericUDAFEvaluator$Mode0" class="java.lang.Enum">
+ * <class>org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode</class>
+ * which needs to be fixed to
+ * <object id="GenericUDAFEvaluator$Mode0" class="org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator$Mode"
+ * method="valueOf">
+ *
+ * Though not elegant, this allows these test cases to pass until we have a better serialization mechanism.
+ *
+ * Did I mention this is test code?
+ *
+ * @param fname the name of the file to fix
+ * @throws Exception in case of IO error
+ */
+ private static void fixXml4JDK7(String fname) throws Exception {
+ String version = System.getProperty("java.version");
+ if (!version.startsWith("1.7")) {
+ return;
+ }
+
+ BufferedReader in = new BufferedReader(new FileReader(fname));
+ BufferedWriter out = new BufferedWriter(new FileWriter(fname + ".orig"));
+ String line = null;
+ while (null != (line = in.readLine())) {
+ out.write(line);
+ out.write('\n');
+ }
+ in.close();
+ out.close();
+
+ in = new BufferedReader(new FileReader(fname + ".orig"));
+ out = new BufferedWriter(new FileWriter(fname));
+
+ while (null != (line = in.readLine())) {
+ if (line.indexOf("<object ") == -1 || line.indexOf("class=") == -1) {
+ out.write(line);
+ } else {
+ StringBuilder sb = new StringBuilder();
+ String prefix = line.substring(0, line.indexOf("<object") + 7);
+ sb.append( prefix );
+ String postfix = line.substring(line.lastIndexOf('"') + 1);
+ String id = getPropertyValue(line, "id");
+ if (id != null)
+ sb.append(" id=" + id);
+ String cls = getPropertyValue(line, "class");
+ assert(cls != null);
+ if (cls.equals("\"java.lang.Enum\"")) {
+ line = in.readLine();
+ cls = "\"" + getElementValue(line, "class") + "\"";
+ sb.append(" class=" + cls + " method=\"valueOf\"" );
+ } else {
+ sb.append(" class=" + cls);
+ }
+
+ sb.append(postfix);
+ out.write(sb.toString());
+ }
+
+ out.write('\n');
+ }
+
+ in.close();
+ out.close();
+ }
+
+ /**
+ * Get the value of a property in line. The returned value has original quotes
+ */
+ private static String getPropertyValue(String line, String name) {
+ int start = line.indexOf( name + "=" );
+ if (start == -1)
+ return null;
+ start += name.length() + 1;
+ int end = line.indexOf("\"", start + 1);
+ return line.substring( start, end + 1 );
+ }
+
+ /**
+ * Get the value of the element in input. (Note: the returned value has no quotes.)
+ */
+ private static String getElementValue(String line, String name) {
+ assert(line.indexOf("<" + name + ">") != -1);
+ int start = line.indexOf("<" + name + ">") + name.length() + 2;
+ int end = line.indexOf("</" + name + ">");
+ return line.substring(start, end);
+ }
+
private void maskPatterns(String[] patterns, String fname) throws Exception {
String maskPattern = "#### A masked pattern was here ####";
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/history/TestHiveHistory.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/history/TestHiveHistory.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/history/TestHiveHistory.java Mon Aug 26 21:42:12 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.histor
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Proxy;
import java.util.LinkedList;
import java.util.Map;
@@ -31,9 +32,9 @@ import org.apache.hadoop.hive.cli.CliSes
import org.apache.hadoop.hive.common.LogUtils;
import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.QTestUtil.QTestSetup;
import org.apache.hadoop.hive.ql.history.HiveHistory.Keys;
import org.apache.hadoop.hive.ql.history.HiveHistory.QueryInfo;
import org.apache.hadoop.hive.ql.history.HiveHistory.TaskInfo;
@@ -75,7 +76,7 @@ public class TestHiveHistory extends Tes
+ tmpdir);
}
}
-
+
conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
// copy the test files into hadoop if required.
@@ -124,8 +125,9 @@ public class TestHiveHistory extends Tes
LogUtils.initHiveLog4j();
} catch (LogInitializationException e) {
}
-
- CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class));
+ HiveConf hconf = new HiveConf(SessionState.class);
+ hconf.setBoolVar(ConfVars.HIVE_SESSION_HISTORY_ENABLED, true);
+ CliSessionState ss = new CliSessionState(hconf);
ss.in = System.in;
try {
ss.out = new PrintStream(System.out, true, "UTF-8");
@@ -179,7 +181,7 @@ public class TestHiveHistory extends Tes
HiveConf conf = new HiveConf(SessionState.class);
conf.set(HiveConf.ConfVars.HIVEHISTORYFILELOC.toString(), actualDir);
SessionState ss = new CliSessionState(conf);
- HiveHistory hiveHistory = new HiveHistory(ss);
+ HiveHistory hiveHistory = new HiveHistoryImpl(ss);
Path actualPath = new Path(actualDir);
if (!fs.exists(actualPath)) {
fail("Query location path is not exist :" + actualPath.toString());
@@ -192,4 +194,38 @@ public class TestHiveHistory extends Tes
}
}
+ /**
+ * Check if HiveHistoryImpl class is returned when hive history is enabled
+ * @throws Exception
+ */
+ public void testHiveHistoryConfigEnabled() throws Exception {
+ HiveConf conf = new HiveConf(SessionState.class);
+ conf.setBoolVar(ConfVars.HIVE_SESSION_HISTORY_ENABLED, true);
+ SessionState ss = new CliSessionState(conf);
+ SessionState.start(ss);
+ HiveHistory hHistory = ss.getHiveHistory();
+ assertEquals("checking hive history class when history is enabled",
+ hHistory.getClass(), HiveHistoryImpl.class);
+ }
+ /**
+ * Check if HiveHistory class is a Proxy class when hive history is disabled
+ * @throws Exception
+ */
+ public void testHiveHistoryConfigDisabled() throws Exception {
+ HiveConf conf = new HiveConf(SessionState.class);
+ conf.setBoolVar(ConfVars.HIVE_SESSION_HISTORY_ENABLED, false);
+ SessionState ss = new CliSessionState(conf);
+ SessionState.start(ss);
+ HiveHistory hHistory = ss.getHiveHistory();
+ assertTrue("checking hive history class when history is disabled",
+ hHistory.getClass() != HiveHistoryImpl.class);
+ System.err.println("hHistory.getClass" + hHistory.getClass());
+ assertTrue("verifying proxy class is used when history is disabled",
+ Proxy.isProxyClass(hHistory.getClass()));
+
+ }
+
+
+
+
}
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java Mon Aug 26 21:42:12 2013
@@ -62,6 +62,7 @@ import org.apache.hadoop.mapred.InputSpl
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.FileSplit;
/**
* TestRCFile.
@@ -573,6 +574,50 @@ public class TestRCFile extends TestCase
splitAfterSync();
}
+ public void testSync() throws IOException {
+ Path testDir = new Path(System.getProperty("test.data.dir", ".")
+ + "/mapred/testsync");
+ Path testFile = new Path(testDir, "test_rcfile");
+ fs.delete(testFile, true);
+ int intervalRecordCount = 500;
+ CompressionCodec codec = null;
+ int writeCount = 2500;
+ Configuration cloneConf = new Configuration(conf);
+ RCFileOutputFormat.setColumnNumber(cloneConf, bytesArray.length);
+ cloneConf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, intervalRecordCount);
+ RCFile.Writer writer = new RCFile.Writer(fs, cloneConf, testFile, null, codec);
+
+ BytesRefArrayWritable bytes = new BytesRefArrayWritable(bytesArray.length);
+ for (int i = 0; i < bytesArray.length; i++) {
+ BytesRefWritable cu = null;
+ cu = new BytesRefWritable(bytesArray[i], 0, bytesArray[i].length);
+ bytes.set(i, cu);
+ }
+ for (int i = 0; i < writeCount; i++) {
+ writer.append(bytes);
+ }
+ writer.close();
+ long fileLen = fs.getFileStatus(testFile).getLen();
+
+ RCFileInputFormat inputFormat = new RCFileInputFormat();
+ JobConf jobconf = new JobConf(cloneConf);
+ jobconf.set("mapred.input.dir", testDir.toString());
+ jobconf.setLong("mapred.min.split.size", fileLen);
+ InputSplit[] splits = inputFormat.getSplits(jobconf, 1);
+ RCFileRecordReader rr = new RCFileRecordReader(jobconf, (FileSplit)splits[0]);
+ long lastSync = 0;
+ for(int i = 0; i < 2500; i++) {
+ rr.sync(i);
+ if(rr.getPos() < lastSync) {
+ String reason = String.format("Sync at offset %d skipped sync block at location %d (returned %d instead)", i-1, rr.getPos(), lastSync);
+ System.out.println(reason);
+ fail(reason);
+ }
+ lastSync = rr.getPos();
+ }
+ rr.close();
+ }
+
private void splitBeforeSync() throws IOException {
writeThenReadByRecordReader(600, 1000, 2, 1, null);
}
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitFieldReader.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitFieldReader.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitFieldReader.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitFieldReader.java Mon Aug 26 21:42:12 2013
@@ -47,7 +47,8 @@ public class TestBitFieldReader {
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
- BitFieldReader in = new BitFieldReader(InStream.create("test", inBuf,
+ BitFieldReader in = new BitFieldReader(InStream.create("test",
+ new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
codec, 500), 1);
for(int i=0; i < COUNT; ++i) {
int x = in.next();
@@ -96,7 +97,8 @@ public class TestBitFieldReader {
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
- BitFieldReader in = new BitFieldReader(InStream.create("test", inBuf,
+ BitFieldReader in = new BitFieldReader(InStream.create("test",
+ new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
null, 500), 3);
for(int i=0; i < COUNT; ++i) {
int x = in.next();
@@ -126,7 +128,8 @@ public class TestBitFieldReader {
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
BitFieldReader in = new BitFieldReader(InStream.create
- ("test", inBuf, null, 100), 1);
+ ("test", new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
+ null, 100), 1);
for(int i=0; i < COUNT; i += 5) {
int x = (int) in.next();
if (i < COUNT/2) {
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java Mon Aug 26 21:42:12 2013
@@ -18,15 +18,8 @@
package org.apache.hadoop.hive.ql.io.orc;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import java.io.BufferedReader;
import java.io.File;
@@ -35,8 +28,14 @@ import java.io.FileReader;
import java.io.PrintStream;
import java.util.Random;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.junit.Before;
+import org.junit.Test;
public class TestFileDump {
@@ -69,9 +68,6 @@ public class TestFileDump {
}
}
- private static final String outputFilename =
- File.separator + "orc-file-dump.out";
-
private static void checkOutput(String expected,
String actual) throws Exception {
BufferedReader eStream =
@@ -114,8 +110,62 @@ public class TestFileDump {
}
writer.close();
PrintStream origOut = System.out;
- FileOutputStream myOut = new FileOutputStream(workDir + File.separator +
- "orc-file-dump.out");
+ String outputFilename = File.separator + "orc-file-dump.out";
+ FileOutputStream myOut = new FileOutputStream(workDir + outputFilename);
+
+ // replace stdout and run command
+ System.setOut(new PrintStream(myOut));
+ FileDump.main(new String[]{testFilePath.toString()});
+ System.out.flush();
+ System.setOut(origOut);
+
+
+ checkOutput(resourceDir + outputFilename, workDir + outputFilename);
+ }
+
+ // Test that if the fraction of rows that have distinct strings is greater than the configured
+ // threshold dictionary encoding is turned off. If dictionary encoding is turned off the length
+ // of the dictionary stream for the column will be 0 in the ORC file dump.
+ @Test
+ public void testDictionaryThreshold() throws Exception {
+ ObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = ObjectInspectorFactory.getReflectionObjectInspector
+ (MyRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ Configuration conf = new Configuration();
+ conf.setFloat(HiveConf.ConfVars.HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD.varname, 0.49f);
+ Writer writer = OrcFile.createWriter(fs, testFilePath, conf, inspector,
+ 100000, CompressionKind.ZLIB, 10000, 10000);
+ Random r1 = new Random(1);
+ String[] words = new String[]{"It", "was", "the", "best", "of", "times,",
+ "it", "was", "the", "worst", "of", "times,", "it", "was", "the", "age",
+ "of", "wisdom,", "it", "was", "the", "age", "of", "foolishness,", "it",
+ "was", "the", "epoch", "of", "belief,", "it", "was", "the", "epoch",
+ "of", "incredulity,", "it", "was", "the", "season", "of", "Light,",
+ "it", "was", "the", "season", "of", "Darkness,", "it", "was", "the",
+ "spring", "of", "hope,", "it", "was", "the", "winter", "of", "despair,",
+ "we", "had", "everything", "before", "us,", "we", "had", "nothing",
+ "before", "us,", "we", "were", "all", "going", "direct", "to",
+ "Heaven,", "we", "were", "all", "going", "direct", "the", "other",
+ "way"};
+ int nextInt = 0;
+ for(int i=0; i < 21000; ++i) {
+ // Write out the same string twice, this guarantees the fraction of rows with
+ // distinct strings is 0.5
+ if (i % 2 == 0) {
+ nextInt = r1.nextInt(words.length);
+ // Append the value of i to the word, this guarantees when an index or word is repeated
+ // the actual string is unique.
+ words[nextInt] += "-" + i;
+ }
+ writer.addRow(new MyRecord(r1.nextInt(), r1.nextLong(),
+ words[nextInt]));
+ }
+ writer.close();
+ PrintStream origOut = System.out;
+ String outputFilename = File.separator + "orc-file-dump-dictionary-threshold.out";
+ FileOutputStream myOut = new FileOutputStream(workDir + outputFilename);
// replace stdout and run command
System.setOut(new PrintStream(myOut));
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInStream.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInStream.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInStream.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInStream.java Mon Aug 26 21:42:12 2013
@@ -20,6 +20,9 @@ package org.apache.hadoop.hive.ql.io.orc
import org.junit.Test;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -40,7 +43,8 @@ public class TestInStream {
}
}
- static class PositionCollector implements PositionProvider, PositionRecorder {
+ static class PositionCollector
+ implements PositionProvider, PositionRecorder {
private List<Long> positions = new ArrayList<Long>();
private int index = 0;
@@ -53,6 +57,22 @@ public class TestInStream {
public void addPosition(long offset) {
positions.add(offset);
}
+
+ public void reset() {
+ index = 0;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder("position: ");
+ for(int i=0; i < positions.size(); ++i) {
+ if (i != 0) {
+ builder.append(", ");
+ }
+ builder.append(positions.get(i));
+ }
+ return builder.toString();
+ }
}
@Test
@@ -73,9 +93,11 @@ public class TestInStream {
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
- InStream in = InStream.create("test", inBuf, null, 100);
- assertEquals("uncompressed stream test base: 0 offset: 0 limit: 1024",
- in.toString());
+ InStream in = InStream.create("test", new ByteBuffer[]{inBuf},
+ new long[]{0}, inBuf.remaining(), null, 100);
+ assertEquals("uncompressed stream test position: 0 length: 1024" +
+ " range: 0 offset: 0 limit: 0",
+ in.toString());
for(int i=0; i < 1024; ++i) {
int x = in.read();
assertEquals(i & 0xff, x);
@@ -103,9 +125,11 @@ public class TestInStream {
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
- InStream in = InStream.create("test", inBuf, codec, 300);
- assertEquals("compressed stream test base: 0 offset: 0 limit: 961",
- in.toString());
+ InStream in = InStream.create("test", new ByteBuffer[]{inBuf},
+ new long[]{0}, inBuf.remaining(), codec, 300);
+ assertEquals("compressed stream test position: 0 length: 961 range: 0" +
+ " offset: 0 limit: 0 range 0 = 0 to 961",
+ in.toString());
for(int i=0; i < 1024; ++i) {
int x = in.read();
assertEquals(i & 0xff, x);
@@ -134,7 +158,8 @@ public class TestInStream {
ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
- InStream in = InStream.create("test", inBuf, codec, 100);
+ InStream in = InStream.create("test", new ByteBuffer[]{inBuf},
+ new long[]{0}, inBuf.remaining(), codec, 100);
byte[] contents = new byte[1024];
try {
in.read(contents);
@@ -148,7 +173,8 @@ public class TestInStream {
inBuf.put((byte) 32);
inBuf.put((byte) 0);
inBuf.flip();
- in = InStream.create("test2", inBuf, codec, 300);
+ in = InStream.create("test2", new ByteBuffer[]{inBuf}, new long[]{0},
+ inBuf.remaining(), codec, 300);
try {
in.read();
fail();
@@ -156,4 +182,132 @@ public class TestInStream {
// EXPECTED
}
}
+
+ @Test
+ public void testDisjointBuffers() throws Exception {
+ OutputCollector collect = new OutputCollector();
+ CompressionCodec codec = new ZlibCodec();
+ OutStream out = new OutStream("test", 400, codec, collect);
+ PositionCollector[] positions = new PositionCollector[1024];
+ DataOutput stream = new DataOutputStream(out);
+ for(int i=0; i < 1024; ++i) {
+ positions[i] = new PositionCollector();
+ out.getPosition(positions[i]);
+ stream.writeInt(i);
+ }
+ out.flush();
+ assertEquals("test", out.toString());
+ assertEquals(1674, collect.buffer.size());
+ ByteBuffer[] inBuf = new ByteBuffer[3];
+ inBuf[0] = ByteBuffer.allocate(500);
+ inBuf[1] = ByteBuffer.allocate(1200);
+ inBuf[2] = ByteBuffer.allocate(500);
+ collect.buffer.setByteBuffer(inBuf[0], 0, 483);
+ collect.buffer.setByteBuffer(inBuf[1], 483, 1625 - 483);
+ collect.buffer.setByteBuffer(inBuf[2], 1625, 1674 - 1625);
+
+ for(int i=0; i < inBuf.length; ++i) {
+ inBuf[i].flip();
+ }
+ InStream in = InStream.create("test", inBuf,
+ new long[]{0,483, 1625}, 1674, codec, 400);
+ assertEquals("compressed stream test position: 0 length: 1674 range: 0" +
+ " offset: 0 limit: 0 range 0 = 0 to 483;" +
+ " range 1 = 483 to 1142; range 2 = 1625 to 49",
+ in.toString());
+ DataInputStream inStream = new DataInputStream(in);
+ for(int i=0; i < 1024; ++i) {
+ int x = inStream.readInt();
+ assertEquals(i, x);
+ }
+ assertEquals(0, in.available());
+ for(int i=1023; i >= 0; --i) {
+ in.seek(positions[i]);
+ assertEquals(i, inStream.readInt());
+ }
+
+ in = InStream.create("test", new ByteBuffer[]{inBuf[1], inBuf[2]},
+ new long[]{483, 1625}, 1674, codec, 400);
+ inStream = new DataInputStream(in);
+ positions[303].reset();
+ in.seek(positions[303]);
+ for(int i=303; i < 1024; ++i) {
+ assertEquals(i, inStream.readInt());
+ }
+
+ in = InStream.create("test", new ByteBuffer[]{inBuf[0], inBuf[2]},
+ new long[]{0, 1625}, 1674, codec, 400);
+ inStream = new DataInputStream(in);
+ positions[1001].reset();
+ for(int i=0; i < 300; ++i) {
+ assertEquals(i, inStream.readInt());
+ }
+ in.seek(positions[1001]);
+ for(int i=1001; i < 1024; ++i) {
+ assertEquals(i, inStream.readInt());
+ }
+ }
+
+ @Test
+ public void testUncompressedDisjointBuffers() throws Exception {
+ OutputCollector collect = new OutputCollector();
+ OutStream out = new OutStream("test", 400, null, collect);
+ PositionCollector[] positions = new PositionCollector[1024];
+ DataOutput stream = new DataOutputStream(out);
+ for(int i=0; i < 1024; ++i) {
+ positions[i] = new PositionCollector();
+ out.getPosition(positions[i]);
+ stream.writeInt(i);
+ }
+ out.flush();
+ assertEquals("test", out.toString());
+ assertEquals(4096, collect.buffer.size());
+ ByteBuffer[] inBuf = new ByteBuffer[3];
+ inBuf[0] = ByteBuffer.allocate(1100);
+ inBuf[1] = ByteBuffer.allocate(2200);
+ inBuf[2] = ByteBuffer.allocate(1100);
+ collect.buffer.setByteBuffer(inBuf[0], 0, 1024);
+ collect.buffer.setByteBuffer(inBuf[1], 1024, 2048);
+ collect.buffer.setByteBuffer(inBuf[2], 3072, 1024);
+
+ for(int i=0; i < inBuf.length; ++i) {
+ inBuf[i].flip();
+ }
+ InStream in = InStream.create("test", inBuf,
+ new long[]{0, 1024, 3072}, 4096, null, 400);
+ assertEquals("uncompressed stream test position: 0 length: 4096" +
+ " range: 0 offset: 0 limit: 0",
+ in.toString());
+ DataInputStream inStream = new DataInputStream(in);
+ for(int i=0; i < 1024; ++i) {
+ int x = inStream.readInt();
+ assertEquals(i, x);
+ }
+ assertEquals(0, in.available());
+ for(int i=1023; i >= 0; --i) {
+ in.seek(positions[i]);
+ assertEquals(i, inStream.readInt());
+ }
+
+ in = InStream.create("test", new ByteBuffer[]{inBuf[1], inBuf[2]},
+ new long[]{1024, 3072}, 4096, null, 400);
+ inStream = new DataInputStream(in);
+ positions[256].reset();
+ in.seek(positions[256]);
+ for(int i=256; i < 1024; ++i) {
+ assertEquals(i, inStream.readInt());
+ }
+
+ in = InStream.create("test", new ByteBuffer[]{inBuf[0], inBuf[2]},
+ new long[]{0, 3072}, 4096, null, 400);
+ inStream = new DataInputStream(in);
+ positions[768].reset();
+ for(int i=0; i < 256; ++i) {
+ assertEquals(i, inStream.readInt());
+ }
+ in.seek(positions[768]);
+ for(int i=768; i < 1024; ++i) {
+ assertEquals(i, inStream.readInt());
+ }
+ }
}
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java Mon Aug 26 21:42:12 2013
@@ -22,6 +22,8 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+import org.apache.hadoop.hive.ql.io.sarg.TestSearchArgumentImpl;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
@@ -58,6 +60,7 @@ import java.nio.ByteBuffer;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
@@ -682,6 +685,7 @@ public class TestOrcFile {
assertEquals(0.0, rows.getProgress(), 0.000001);
assertEquals(true, rows.hasNext());
row = (OrcStruct) rows.next(null);
+ assertEquals(1, rows.getRowNumber());
inspector = reader.getObjectInspector();
assertEquals("struct<time:timestamp,union:uniontype<int,string>,decimal:decimal>",
inspector.getTypeName());
@@ -692,6 +696,7 @@ public class TestOrcFile {
assertEquals(new IntWritable(42), union.getObject());
assertEquals(new HiveDecimal("12345678.6547456"), row.getFieldValue(2));
row = (OrcStruct) rows.next(row);
+ assertEquals(2, rows.getRowNumber());
assertEquals(Timestamp.valueOf("2000-03-20 12:00:00.123456789"),
row.getFieldValue(0));
assertEquals(1, union.getTag());
@@ -905,6 +910,40 @@ public class TestOrcFile {
compareList(expectedList, actualList);
compareList(expected.list, (List) row.getFieldValue(10));
}
+ rows.close();
+ Iterator<StripeInformation> stripeIterator =
+ reader.getStripes().iterator();
+ long offsetOfStripe2 = 0;
+ long offsetOfStripe4 = 0;
+ long lastRowOfStripe2 = 0;
+ for(int i = 0; i < 5; ++i) {
+ StripeInformation stripe = stripeIterator.next();
+ if (i < 2) {
+ lastRowOfStripe2 += stripe.getNumberOfRows();
+ } else if (i == 2) {
+ offsetOfStripe2 = stripe.getOffset();
+ lastRowOfStripe2 += stripe.getNumberOfRows() - 1;
+ } else if (i == 4) {
+ offsetOfStripe4 = stripe.getOffset();
+ }
+ }
+ boolean[] columns = new boolean[reader.getStatistics().length];
+ columns[5] = true; // long colulmn
+ columns[9] = true; // text column
+ rows = reader.rows(offsetOfStripe2, offsetOfStripe4 - offsetOfStripe2,
+ columns);
+ rows.seekToRow(lastRowOfStripe2);
+ for(int i = 0; i < 2; ++i) {
+ row = (OrcStruct) rows.next(row);
+ BigRow expected = createRandomRow(intValues, doubleValues,
+ stringValues, byteValues, words,
+ (int) (lastRowOfStripe2 + i));
+
+ assertEquals(expected.long1.longValue(),
+ ((LongWritable) row.getFieldValue(4)).get());
+ assertEquals(expected.string1, row.getFieldValue(8));
+ }
+ rows.close();
}
private void compareInner(InnerStruct expect,
@@ -993,7 +1032,7 @@ public class TestOrcFile {
ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
MyMemoryManager memory = new MyMemoryManager(conf, 10000, 0.1);
- Writer writer = new WriterImpl(fs, testFilePath, inspector,
+ Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
50000, CompressionKind.NONE, 100, 0, memory);
assertEquals(testFilePath, memory.path);
for(int i=0; i < 2500; ++i) {
@@ -1011,4 +1050,86 @@ public class TestOrcFile {
assertEquals(25, i);
assertEquals(2500, reader.getNumberOfRows());
}
+
+ @Test
+ public void testPredicatePushdown() throws Exception {
+ ObjectInspector inspector;
+ synchronized (TestOrcFile.class) {
+ inspector = ObjectInspectorFactory.getReflectionObjectInspector
+ (InnerStruct.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ }
+ Writer writer = OrcFile.createWriter(fs, testFilePath, conf, inspector,
+ 400000L, CompressionKind.NONE, 500, 1000);
+ for(int i=0; i < 3500; ++i) {
+ writer.addRow(new InnerStruct(i*300, Integer.toHexString(10*i)));
+ }
+ writer.close();
+ Reader reader = OrcFile.createReader(fs, testFilePath);
+ assertEquals(3500, reader.getNumberOfRows());
+
+ SearchArgument sarg = SearchArgument.FACTORY.newBuilder()
+ .startAnd()
+ .startNot()
+ .lessThan("int1", 300000)
+ .end()
+ .lessThan("int1", 600000)
+ .end()
+ .build();
+ RecordReader rows = reader.rows(0L, Long.MAX_VALUE,
+ new boolean[]{true, true, true}, sarg,
+ new String[]{null, "int1", "string1"});
+ assertEquals(1000L, rows.getRowNumber());
+ OrcStruct row = null;
+ for(int i=1000; i < 2000; ++i) {
+ assertTrue(rows.hasNext());
+ row = (OrcStruct) rows.next(row);
+ assertEquals(300 * i, ((IntWritable) row.getFieldValue(0)).get());
+ assertEquals(Integer.toHexString(10*i), row.getFieldValue(1).toString());
+ }
+ assertTrue(!rows.hasNext());
+ assertEquals(3500, rows.getRowNumber());
+
+ // look through the file with no rows selected
+ sarg = SearchArgument.FACTORY.newBuilder()
+ .startAnd()
+ .lessThan("int1", 0)
+ .end()
+ .build();
+ rows = reader.rows(0L, Long.MAX_VALUE,
+ new boolean[]{true, true, true}, sarg,
+ new String[]{null, "int1", "string1"});
+ assertEquals(3500L, rows.getRowNumber());
+ assertTrue(!rows.hasNext());
+
+ // select first 100 and last 100 rows
+ sarg = SearchArgument.FACTORY.newBuilder()
+ .startOr()
+ .lessThan("int1", 300 * 100)
+ .startNot()
+ .lessThan("int1", 300 * 3400)
+ .end()
+ .end()
+ .build();
+ rows = reader.rows(0L, Long.MAX_VALUE,
+ new boolean[]{true, true, true}, sarg,
+ new String[]{null, "int1", "string1"});
+ row = null;
+ for(int i=0; i < 1000; ++i) {
+ assertTrue(rows.hasNext());
+ assertEquals(i, rows.getRowNumber());
+ row = (OrcStruct) rows.next(row);
+ assertEquals(300 * i, ((IntWritable) row.getFieldValue(0)).get());
+ assertEquals(Integer.toHexString(10*i), row.getFieldValue(1).toString());
+ }
+ for(int i=3000; i < 3500; ++i) {
+ assertTrue(rows.hasNext());
+ assertEquals(i, rows.getRowNumber());
+ row = (OrcStruct) rows.next(row);
+ assertEquals(300 * i, ((IntWritable) row.getFieldValue(0)).get());
+ assertEquals(Integer.toHexString(10*i), row.getFieldValue(1).toString());
+ }
+ assertTrue(!rows.hasNext());
+ assertEquals(3500, rows.getRowNumber());
+ }
}
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcNullOptimization.java Mon Aug 26 21:42:12 2013
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hive.ql.io.orc;
import static junit.framework.Assert.assertEquals;
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java Mon Aug 26 21:42:12 2013
@@ -46,7 +46,7 @@ public class TestRunLengthByteReader {
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
RunLengthByteReader in = new RunLengthByteReader(InStream.create("test",
- inBuf, null, 100));
+ new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(), null, 100));
for(int i=0; i < 2048; ++i) {
int x = in.next() & 0xff;
if (i < 1024) {
@@ -88,7 +88,7 @@ public class TestRunLengthByteReader {
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
RunLengthByteReader in = new RunLengthByteReader(InStream.create("test",
- inBuf, codec, 500));
+ new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(), codec, 500));
for(int i=0; i < 2048; ++i) {
int x = in.next() & 0xff;
if (i < 1024) {
@@ -125,7 +125,7 @@ public class TestRunLengthByteReader {
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
RunLengthByteReader in = new RunLengthByteReader(InStream.create("test",
- inBuf, null, 100));
+ new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(), null, 100));
for(int i=0; i < 2048; i += 10) {
int x = in.next() & 0xff;
if (i < 1024) {
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java Mon Aug 26 21:42:12 2013
@@ -54,7 +54,8 @@ public class TestRunLengthIntegerReader
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
RunLengthIntegerReader in = new RunLengthIntegerReader(InStream.create
- ("test", inBuf, codec, 1000), true);
+ ("test", new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
+ codec, 1000), true);
for(int i=0; i < 2048; ++i) {
int x = (int) in.next();
if (i < 1024) {
@@ -105,7 +106,8 @@ public class TestRunLengthIntegerReader
collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
inBuf.flip();
RunLengthIntegerReader in = new RunLengthIntegerReader(InStream.create
- ("test", inBuf, null, 100), true);
+ ("test", new ByteBuffer[]{inBuf}, new long[]{0}, inBuf.remaining(),
+ null, 100), true);
for(int i=0; i < 2048; i += 10) {
int x = (int) in.next();
if (i < 1024) {
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateAdd.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateAdd.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateAdd.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateAdd.java Mon Aug 26 21:42:12 2013
@@ -18,12 +18,12 @@
package org.apache.hadoop.hive.ql.udf;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
+import java.util.TimeZone;
import junit.framework.TestCase;
-import java.util.TimeZone;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
/**
* JUnit test for UDFDateAdd.
@@ -37,7 +37,7 @@ public class TestUDFDateAdd extends Test
* on 2009-10-31.
*/
public void testFallBack() throws Exception {
- // set the default time zone so that the dates cover
+ // set the default time zone so that the dates cover
// the zone's daylight saving time adjustment (2009-10-31)
// from daylight to standard time
TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"));
@@ -55,7 +55,7 @@ public class TestUDFDateAdd extends Test
* on 2010-03-14.
*/
public void testSpringAhead() throws Exception {
- // set the default time zone so that the dates cover
+ // set the default time zone so that the dates cover
// the zone's daylight saving time adjustment (2010-03-14)
// from standard to daylight time
TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"));
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestTranslate.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestTranslate.java?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestTranslate.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestTranslate.java Mon Aug 26 21:42:12 2013
@@ -35,7 +35,7 @@ import org.apache.hadoop.io.Text;
* Mimics oracle's function translate(str1, str2, str3).
*/
public class GenericUDFTestTranslate extends GenericUDF {
- ObjectInspector[] argumentOIs;
+ private transient ObjectInspector[] argumentOIs;
/**
* Return a corresponding ordinal from an integer.
Modified: hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithNoGBYNoWindowing.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithNoGBYNoWindowing.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithNoGBYNoWindowing.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithNoGBYNoWindowing.q Mon Aug 26 21:42:12 2013
@@ -12,11 +12,9 @@ CREATE TABLE part(
p_comment STRING
);
-LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part;
-
-- testHavingLeadWithNoGBYNoWindowing
select p_mfgr,p_name, p_size
from part
-having lead(p_size, 1) <= p_size
+having lead(p_size, 1) over() <= p_size
distribute by p_mfgr
sort by p_name;
Modified: hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithPTF.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithPTF.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithPTF.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_HavingLeadWithPTF.q Mon Aug 26 21:42:12 2013
@@ -17,6 +17,6 @@ select p_mfgr,p_name, p_size
from noop(on part
partition by p_mfgr
order by p_name)
-having lead(p_size, 1) <= p_size
+having lead(p_size, 1) over() <= p_size
distribute by p_mfgr
sort by p_name;
Modified: hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_InvalidValueBoundary.q Mon Aug 26 21:42:12 2013
@@ -16,6 +16,6 @@ CREATE TABLE part(
-- testInvalidValueBoundary
select p_mfgr,p_name, p_size,
sum(p_size) over (w1) as s ,
-dense_rank() as dr
+dense_rank() over(w1) as dr
from part
window w1 as (partition by p_mfgr order by p_complex range between 2 preceding and current row);
Modified: hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_WhereWithRankCond.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_WhereWithRankCond.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_WhereWithRankCond.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientnegative/ptf_negative_WhereWithRankCond.q Mon Aug 26 21:42:12 2013
@@ -12,11 +12,9 @@ CREATE TABLE part(
p_comment STRING
);
-LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part;
-
-- testWhereWithRankCond
select p_mfgr,p_name, p_size,
-rank() as r
+rank() over() as r
from part
where r < 4
distribute by p_mfgr
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/ba_table_udfs.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/ba_table_udfs.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/ba_table_udfs.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/ba_table_udfs.q Mon Aug 26 21:42:12 2013
@@ -1,27 +1,36 @@
USE default;
CREATE TABLE dest1(bytes1 BINARY,
- bytes2 BINARY);
+ bytes2 BINARY,
+ string STRING);
FROM src INSERT OVERWRITE TABLE dest1
SELECT
CAST(key AS BINARY),
- CAST(value AS BINARY)
+ CAST(value AS BINARY),
+ value
ORDER BY value
LIMIT 100;
--Add in a null row for good measure
-INSERT INTO TABLE dest1 SELECT NULL, NULL FROM dest1 LIMIT 1;
+INSERT INTO TABLE dest1 SELECT NULL, NULL, NULL FROM dest1 LIMIT 1;
-- this query tests all the udfs provided to work with binary types
SELECT
bytes1,
bytes2,
+ string,
LENGTH(bytes1),
CONCAT(bytes1, bytes2),
SUBSTR(bytes2, 1, 4),
SUBSTR(bytes2, 3),
- SUBSTR(bytes2, -4, 3)
+ SUBSTR(bytes2, -4, 3),
+ HEX(bytes1),
+ UNHEX(HEX(bytes1)),
+ BASE64(bytes1),
+ UNBASE64(BASE64(bytes1)),
+ HEX(ENCODE(string, 'US-ASCII')),
+ DECODE(ENCODE(string, 'US-ASCII'), 'US-ASCII')
FROM dest1
ORDER BY bytes2;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/create_udaf.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/create_udaf.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/create_udaf.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/create_udaf.q Mon Aug 26 21:42:12 2013
@@ -9,4 +9,11 @@ FROM src INSERT OVERWRITE TABLE dest1 SE
SELECT dest1.* FROM dest1;
+-- cover all the other value types:
+SELECT test_max(CAST(length(src.value) AS SMALLINT)) FROM src;
+SELECT test_max(CAST(length(src.value) AS BIGINT)) FROM src;
+SELECT test_max(CAST(length(src.value) AS DOUBLE)) FROM src;
+SELECT test_max(CAST(length(src.value) AS FLOAT)) FROM src;
+SELECT test_max(substr(src.value,5)) FROM src;
+
DROP TEMPORARY FUNCTION test_max;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/ctas_colname.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/ctas_colname.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/ctas_colname.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/ctas_colname.q Mon Aug 26 21:42:12 2013
@@ -16,10 +16,10 @@ describe formatted x4;
select * from x4 order by key, value, rr;
explain
-create table x5 as select *, lead(key,1) over(partition by key order by value) from src limit 20;
-create table x5 as select *, lead(key,1) over(partition by key order by value) from src limit 20;
+create table x5 as select *, lead(key,1) over(partition by key order by value) as lead1 from src limit 20;
+create table x5 as select *, lead(key,1) over(partition by key order by value) as lead1 from src limit 20;
describe formatted x5;
-select * from x5 order by key, value, tok_windowspec;
+select * from x5 order by key, value, lead1;
-- sub queries
explain
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/list_bucket_dml_7.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/list_bucket_dml_7.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/list_bucket_dml_7.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/list_bucket_dml_7.q Mon Aug 26 21:42:12 2013
@@ -64,7 +64,7 @@ set hive.input.format=org.apache.hadoop.
explain extended
select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484';
select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484';
-select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484';
+select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484' order by hr;
-- clean up
drop table list_bucketing_dynamic_part;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/list_bucket_dml_8.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/list_bucket_dml_8.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/list_bucket_dml_8.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/list_bucket_dml_8.q Mon Aug 26 21:42:12 2013
@@ -81,7 +81,7 @@ select count(*) from list_bucketing_dyna
explain extended
select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484';
select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484';
-select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484';
+select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484' order by hr;
-- clean up
drop table list_bucketing_dynamic_part;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/nonblock_op_deduplicate.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/nonblock_op_deduplicate.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/nonblock_op_deduplicate.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/nonblock_op_deduplicate.q Mon Aug 26 21:42:12 2013
@@ -12,7 +12,7 @@ FROM (SELECT tmp2.key as key, tmp2.value
FROM src1) tmp1 ) tmp2
JOIN (SELECT count(*) as count
FROM src1) tmp3
- ) tmp4;
+ ) tmp4 order by key, value, count;
SELECT tmp4.key as key, tmp4.value as value, tmp4.count as count
FROM (SELECT tmp2.key as key, tmp2.value as value, tmp3.count as count
@@ -21,7 +21,7 @@ FROM (SELECT tmp2.key as key, tmp2.value
FROM src1) tmp1 ) tmp2
JOIN (SELECT count(*) as count
FROM src1) tmp3
- ) tmp4;
+ ) tmp4 order by key, value, count;
set hive.auto.convert.join=true;
-- Then, we convert the join to MapJoin.
@@ -33,7 +33,7 @@ FROM (SELECT tmp2.key as key, tmp2.value
FROM src1) tmp1 ) tmp2
JOIN (SELECT count(*) as count
FROM src1) tmp3
- ) tmp4;
+ ) tmp4 order by key, value, count;
SELECT tmp4.key as key, tmp4.value as value, tmp4.count as count
FROM (SELECT tmp2.key as key, tmp2.value as value, tmp3.count as count
@@ -42,4 +42,4 @@ FROM (SELECT tmp2.key as key, tmp2.value
FROM src1) tmp1 ) tmp2
JOIN (SELECT count(*) as count
FROM src1) tmp3
- ) tmp4;
+ ) tmp4 order by key, value, count;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/partition_date.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/partition_date.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/partition_date.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/partition_date.q Mon Aug 26 21:42:12 2013
@@ -12,7 +12,7 @@ insert overwrite table partition_date_1
select * from src limit 11;
select distinct dt from partition_date_1;
-select *, cast(dt as timestamp) from partition_date_1 where dt = '2000-01-01' and region = 2;
+select *, cast(dt as timestamp) from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value;
-- 15
select count(*) from partition_date_1 where dt = date '2000-01-01';
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/partition_date2.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/partition_date2.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/partition_date2.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/partition_date2.q Mon Aug 26 21:42:12 2013
@@ -3,12 +3,10 @@ drop table partition_date2_1;
create table partition_date2_1 (key string, value string) partitioned by (dt date, region int);
-- test date literal syntax
-insert overwrite table partition_date2_1 partition(dt=date '2000-01-01', region=1)
- select * from src limit 1;
-insert overwrite table partition_date2_1 partition(dt=date '2000-01-01', region=2)
- select * from src limit 1;
-insert overwrite table partition_date2_1 partition(dt=date '1999-01-01', region=2)
- select * from src limit 1;
+from (select * from src limit 1) x
+insert overwrite table partition_date2_1 partition(dt=date '2000-01-01', region=1) select *
+insert overwrite table partition_date2_1 partition(dt=date '2000-01-01', region=2) select *
+insert overwrite table partition_date2_1 partition(dt=date '1999-01-01', region=2) select *;
select distinct dt from partition_date2_1;
select * from partition_date2_1;
@@ -44,7 +42,7 @@ describe extended partition_date2_1 par
insert overwrite table partition_date2_1 partition(dt=date '1980-01-02', region=3)
select * from src limit 2;
-select * from partition_date2_1;
+select * from partition_date2_1 order by key,value,dt,region;
-- alter table set location
alter table partition_date2_1 partition(dt=date '1980-01-02', region=3)
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/ptf.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/ptf.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/ptf.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/ptf.q Mon Aug 26 21:42:12 2013
@@ -196,7 +196,7 @@ rank() over (distribute by p_mfgr sort b
dense_rank() over (distribute by p_mfgr sort by p_name) as dr,
sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s
INSERT OVERWRITE TABLE part_5 select p_mfgr,p_name, p_size,
-round(sum(p_size),1) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row) as s2,
+round(sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row),1) as s2,
rank() over (distribute by p_mfgr sort by p_mfgr, p_name) as r,
dense_rank() over (distribute by p_mfgr sort by p_mfgr, p_name) as dr,
cume_dist() over (distribute by p_mfgr sort by p_mfgr, p_name) as cud,
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/split_sample.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/split_sample.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/split_sample.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/split_sample.q Mon Aug 26 21:42:12 2013
@@ -110,3 +110,6 @@ select key from ss_src2 tablesample(10 R
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
-- ROW type works with other input formats (others, don't)
select count(1) from ss_src2 tablesample(10 ROWS);
+
+--HIVE-5061 row sampling in sub-query
+select * from (select * from src TABLESAMPLE (1 ROWS)) x;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/udf4.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/udf4.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/udf4.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/udf4.q Mon Aug 26 21:42:12 2013
@@ -3,6 +3,49 @@ CREATE TABLE dest1(c1 STRING) STORED AS
FROM src INSERT OVERWRITE TABLE dest1 SELECT ' abc ' WHERE src.key = 86;
EXPLAIN
-SELECT round(1.0), round(1.5), round(-1.5), floor(1.0), floor(1.5), floor(-1.5), sqrt(1.0), sqrt(-1.0), sqrt(0.0), ceil(1.0), ceil(1.5), ceil(-1.5), ceiling(1.0), rand(3), +3, -3, 1++2, 1+-2, ~1 FROM dest1;
+SELECT round(1.0), round(1.5), round(-1.5), floor(1.0), floor(1.5), floor(-1.5), sqrt(1.0), sqrt(-1.0), sqrt(0.0), ceil(1.0), ceil(1.5), ceil(-1.5), ceiling(1.0), rand(3), +3, -3, 1++2, 1+-2,
-SELECT round(1.0), round(1.5), round(-1.5), floor(1.0), floor(1.5), floor(-1.5), sqrt(1.0), sqrt(-1.0), sqrt(0.0), ceil(1.0), ceil(1.5), ceil(-1.5), ceiling(1.0), rand(3), +3, -3, 1++2, 1+-2, ~1 FROM dest1;
+~1,
+~CAST(1 AS TINYINT),
+~CAST(1 AS SMALLINT),
+~CAST(1 AS BIGINT),
+
+CAST(1 AS TINYINT) & CAST(2 AS TINYINT),
+CAST(1 AS SMALLINT) & CAST(2 AS SMALLINT),
+1 & 2,
+CAST(1 AS BIGINT) & CAST(2 AS BIGINT),
+
+CAST(1 AS TINYINT) | CAST(2 AS TINYINT),
+CAST(1 AS SMALLINT) | CAST(2 AS SMALLINT),
+1 | 2,
+CAST(1 AS BIGINT) | CAST(2 AS BIGINT),
+
+CAST(1 AS TINYINT) ^ CAST(3 AS TINYINT),
+CAST(1 AS SMALLINT) ^ CAST(3 AS SMALLINT),
+1 ^ 3,
+CAST(1 AS BIGINT) ^ CAST(3 AS BIGINT)
+
+FROM dest1;
+
+SELECT round(1.0), round(1.5), round(-1.5), floor(1.0), floor(1.5), floor(-1.5), sqrt(1.0), sqrt(-1.0), sqrt(0.0), ceil(1.0), ceil(1.5), ceil(-1.5), ceiling(1.0), rand(3), +3, -3, 1++2, 1+-2,
+~1,
+~CAST(1 AS TINYINT),
+~CAST(1 AS SMALLINT),
+~CAST(1 AS BIGINT),
+
+CAST(1 AS TINYINT) & CAST(2 AS TINYINT),
+CAST(1 AS SMALLINT) & CAST(2 AS SMALLINT),
+1 & 2,
+CAST(1 AS BIGINT) & CAST(2 AS BIGINT),
+
+CAST(1 AS TINYINT) | CAST(2 AS TINYINT),
+CAST(1 AS SMALLINT) | CAST(2 AS SMALLINT),
+1 | 2,
+CAST(1 AS BIGINT) | CAST(2 AS BIGINT),
+
+CAST(1 AS TINYINT) ^ CAST(3 AS TINYINT),
+CAST(1 AS SMALLINT) ^ CAST(3 AS SMALLINT),
+1 ^ 3,
+CAST(1 AS BIGINT) ^ CAST(3 AS BIGINT)
+
+FROM dest1;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/udf_pmod.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/udf_pmod.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/udf_pmod.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/udf_pmod.q Mon Aug 26 21:42:12 2013
@@ -9,3 +9,12 @@ FROM src LIMIT 1;
SELECT pmod(100,19), pmod(50,125), pmod(300,15)
FROM src LIMIT 1;
+
+SELECT pmod(CAST(-100 AS TINYINT),CAST(9 AS TINYINT)), pmod(CAST(-50 AS TINYINT),CAST(101 AS TINYINT)), pmod(CAST(-100 AS TINYINT),CAST(29 AS TINYINT)) FROM src LIMIT 1;
+SELECT pmod(CAST(-100 AS SMALLINT),CAST(9 AS SMALLINT)), pmod(CAST(-50 AS SMALLINT),CAST(101 AS SMALLINT)), pmod(CAST(-100 AS SMALLINT),CAST(29 AS SMALLINT)) FROM src LIMIT 1;
+SELECT pmod(CAST(-100 AS BIGINT),CAST(9 AS BIGINT)), pmod(CAST(-50 AS BIGINT),CAST(101 AS BIGINT)), pmod(CAST(-100 AS BIGINT),CAST(29 AS BIGINT)) FROM src LIMIT 1;
+
+SELECT pmod(CAST(-100.91 AS FLOAT),CAST(9.8 AS FLOAT)), pmod(CAST(-50.1 AS FLOAT),CAST(101.8 AS FLOAT)), pmod(CAST(-100.91 AS FLOAT),CAST(29.75 AS FLOAT)) FROM src LIMIT 1;
+SELECT pmod(CAST(-100.91 AS DOUBLE),CAST(9.8 AS DOUBLE)), pmod(CAST(-50.1 AS DOUBLE),CAST(101.8 AS DOUBLE)), pmod(CAST(-100.91 AS DOUBLE),CAST(29.75 AS DOUBLE)) FROM src LIMIT 1;
+SELECT pmod(CAST(-100.91 AS DECIMAL),CAST(9.8 AS DECIMAL)), pmod(CAST(-50.1 AS DECIMAL),CAST(101.8 AS DECIMAL)), pmod(CAST(-100.91 AS DECIMAL),CAST(29.75 AS DECIMAL)) FROM src LIMIT 1;
+
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/udf_sentences.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/udf_sentences.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/udf_sentences.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/udf_sentences.q Mon Aug 26 21:42:12 2013
@@ -1,7 +1,7 @@
CREATE TABLE sent_tmp (val array<string>);
CREATE TABLE sent_tmp2 (val string);
INSERT OVERWRITE TABLE sent_tmp
-SELECT explode(sentences(unhex("486976652065737420756E20657863656C6C656E74206F7574696C20706F7572206C65732072657175C3AA74657320646520646F6E6EC3A965732C20657420706575742DC3AA74726520706C757320706F6C7976616C656E7420717565206C612074726164756374696F6E206175746F6D61746971756521206C6120706F6E6374756174696F6E206D756C7469706C65732C206465732070687261736573206D616C20666F726DC3A96573202E2E2E20636F6E667573696F6E202D20657420706F757274616E742063652055444620666F6E6374696F6E6E6520656E636F72652121"), "fr")) AS val FROM src LIMIT 3;
+SELECT explode(sentences(decode(unhex("486976652065737420756E20657863656C6C656E74206F7574696C20706F7572206C65732072657175C3AA74657320646520646F6E6EC3A965732C20657420706575742DC3AA74726520706C757320706F6C7976616C656E7420717565206C612074726164756374696F6E206175746F6D61746971756521206C6120706F6E6374756174696F6E206D756C7469706C65732C206465732070687261736573206D616C20666F726DC3A96573202E2E2E20636F6E667573696F6E202D20657420706F757274616E742063652055444620666F6E6374696F6E6E6520656E636F72652121"), "UTF-8"), "fr")) AS val FROM src LIMIT 3;
INSERT OVERWRITE TABLE sent_tmp2
SELECT explode(val) AS val FROM sent_tmp;
SELECT hex(val) AS value FROM sent_tmp2 ORDER BY value ASC;
@@ -12,7 +12,7 @@ DROP TABLE sent_tmp2;
CREATE TABLE sent_tmp (val array<string>);
CREATE TABLE sent_tmp2 (val string);
INSERT OVERWRITE TABLE sent_tmp
-SELECT explode(sentences(unhex("48697665206973742065696E2061757367657A656963686E65746573205765726B7A6575672066C3BC7220646965204162667261676520766F6E20446174656E2C20756E64207669656C6C6569636874207669656C736569746967657220616C7320646965206D61736368696E656C6C6520C39C6265727365747A756E6721204D756C7469706C652C207363686C6563687420676562696C646574656E2053C3A4747A65202E2E2E205665727765636873656C756E6720496E74657270756E6B74696F6E202D20756E6420646F636820697374206469657365205544462066756E6B74696F6E6965727420696D6D6572206E6F63682121"), "de")) AS val FROM src LIMIT 3;
+SELECT explode(sentences(decode(unhex("48697665206973742065696E2061757367657A656963686E65746573205765726B7A6575672066C3BC7220646965204162667261676520766F6E20446174656E2C20756E64207669656C6C6569636874207669656C736569746967657220616C7320646965206D61736368696E656C6C6520C39C6265727365747A756E6721204D756C7469706C652C207363686C6563687420676562696C646574656E2053C3A4747A65202E2E2E205665727765636873656C756E6720496E74657270756E6B74696F6E202D20756E6420646F636820697374206469657365205544462066756E6B74696F6E6965727420696D6D6572206E6F63682121"), "UTF-8"), "de")) AS val FROM src LIMIT 3;
INSERT OVERWRITE TABLE sent_tmp2
SELECT explode(val) AS val FROM sent_tmp;
SELECT hex(val) AS value FROM sent_tmp2 ORDER BY value ASC;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/windowing.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/windowing.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/windowing.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/windowing.q Mon Aug 26 21:42:12 2013
@@ -218,14 +218,14 @@ group by p_mfgr, p_brand;
select *
from (
select p_mfgr, p_brand, s,
-round(sum(s),2) over w1 as s1
+round(sum(s) over w1 , 2) as s1
from mfgr_price_view
window w1 as (distribute by p_mfgr sort by p_mfgr )
) sq
order by p_mfgr, p_brand;
select p_mfgr, p_brand, s,
-round(sum(s),2) over w1 as s1
+round(sum(s) over w1 ,2) as s1
from mfgr_price_view
window w1 as (distribute by p_mfgr sort by p_brand rows between 2 preceding and current row);
@@ -283,7 +283,7 @@ select p_mfgr,p_name, p_size,
rank() over(distribute by p_mfgr sort by p_name) as r,
dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
-round(sum(p_size),1) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row) as s2,
+round(sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row),1) as s2,
first_value(p_size) over w1 as fv1
window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
INSERT OVERWRITE TABLE part_3
@@ -387,10 +387,10 @@ from part;
-- 37. testPartitioningVariousForms
select p_mfgr,
-round(sum(p_retailprice),2) over (partition by p_mfgr order by p_mfgr) as s1,
+round(sum(p_retailprice) over (partition by p_mfgr order by p_mfgr),2) as s1,
min(p_retailprice) over (partition by p_mfgr) as s2,
max(p_retailprice) over (distribute by p_mfgr sort by p_mfgr) as s3,
-round(avg(p_retailprice),2) over (distribute by p_mfgr) as s4,
+round(avg(p_retailprice) over (distribute by p_mfgr),2) as s4,
count(p_retailprice) over (cluster by p_mfgr ) as s5
from part;
@@ -428,7 +428,7 @@ select p_mfgr, p_name, p_size,
-- 44. testOverNoPartitionSingleAggregate
select p_name, p_retailprice,
-round(avg(p_retailprice),2) over()
+round(avg(p_retailprice) over(),2)
from part
order by p_name;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/windowing_expressions.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/windowing_expressions.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/windowing_expressions.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/windowing_expressions.q Mon Aug 26 21:42:12 2013
@@ -35,17 +35,15 @@ create table over10k(
load data local inpath '../data/files/over10k' into table over10k;
select p_mfgr, p_retailprice, p_size,
-round(sum(p_retailprice),2) = round(sum(lag(p_retailprice,1,0.0)) + last_value(p_retailprice),2)
- over(distribute by p_mfgr sort by p_retailprice),
-max(p_retailprice) - min(p_retailprice) = last_value(p_retailprice) - first_value(p_retailprice)
- over(distribute by p_mfgr sort by p_retailprice)
+round(sum(p_retailprice) over w1 , 2) = round(sum(lag(p_retailprice,1,0.0)) over w1 + last_value(p_retailprice) over w1 , 2),
+max(p_retailprice) over w1 - min(p_retailprice) over w1 = last_value(p_retailprice) over w1 - first_value(p_retailprice) over w1
from part
+window w1 as (distribute by p_mfgr sort by p_retailprice)
;
-
select p_mfgr, p_retailprice, p_size,
rank() over (distribute by p_mfgr sort by p_retailprice) as r,
sum(p_retailprice) over (distribute by p_mfgr sort by p_retailprice rows between unbounded preceding and current row) as s2,
-sum(p_retailprice) - 5 over (distribute by p_mfgr sort by p_retailprice rows between unbounded preceding and current row) as s1
+sum(p_retailprice) over (distribute by p_mfgr sort by p_retailprice rows between unbounded preceding and current row) -5 as s1
from part
;
@@ -66,7 +64,7 @@ select * from t1 limit 3;
select * from t2 limit 3;
select p_mfgr, p_retailprice, p_size,
-round(sum(p_retailprice),2) + 50.0 = round(sum(lag(p_retailprice,1,50.0)) + last_value(p_retailprice),2)
- over(distribute by p_mfgr sort by p_retailprice)
+round(sum(p_retailprice) over w1 , 2) + 50.0 = round(sum(lag(p_retailprice,1,50.0)) over w1 + (last_value(p_retailprice) over w1),2)
from part
+window w1 as (distribute by p_mfgr sort by p_retailprice)
limit 11;
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/windowing_windowspec.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/windowing_windowspec.q?rev=1517707&r1=1517706&r2=1517707&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/windowing_windowspec.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/windowing_windowspec.q Mon Aug 26 21:42:12 2013
@@ -31,4 +31,6 @@ select s, sum(i) over(partition by ts or
select f, sum(f) over (partition by ts order by f range between unbounded preceding and current row) from over10k limit 100;
+select s, i, round(avg(d) over (partition by s order by i) / 10.0 , 2) from over10k limit 7;
+select s, i, round((avg(d) over w1 + 10.0) - (avg(d) over w1 - 10.0),2) from over10k window w1 as (partition by s order by i) limit 7;