You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/07/09 11:07:43 UTC
svn commit: r1501145 [4/13] - in /hive/branches/vectorization: ./
beeline/src/java/org/apache/hive/beeline/
beeline/src/test/org/apache/hive/beeline/src/test/
cli/src/java/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/common/metric...
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java Tue Jul 9 09:07:35 2013
@@ -142,6 +142,17 @@ public class ExprNodeGenericFuncEvaluato
}
@Override
+ public boolean isStateful() {
+ boolean result = FunctionRegistry.isStateful(genericUDF);
+ for (ExprNodeEvaluator child : children) {
+ if(result = result || child.isStateful()) {
+ return result;
+ }
+ }
+ return result;
+ }
+
+ @Override
protected Object _evaluate(Object row, int version) throws HiveException {
rowObject = row;
if (ObjectInspectorUtils.isConstantObjectInspector(outputOI) &&
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Tue Jul 9 09:07:35 2013
@@ -493,11 +493,15 @@ public class FetchOperator implements Se
public boolean pushRow() throws IOException, HiveException {
InspectableObject row = getNextRow();
if (row != null) {
- operator.process(row.o, 0);
+ pushRow(row);
}
return row != null;
}
+ protected void pushRow(InspectableObject row) throws HiveException {
+ operator.process(row.o, 0);
+ }
+
private transient final InspectableObject inspectable = new InspectableObject();
/**
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java Tue Jul 9 09:07:35 2013
@@ -163,19 +163,6 @@ public class FetchTask extends Task<Fetc
return "FETCH";
}
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- String s = work.getTblDir();
- if ((s != null) && ctx.isMRTmpFileURI(s)) {
- work.setTblDir(ctx.localizeMRTmpFileURI(s));
- }
-
- ArrayList<String> ls = work.getPartDir();
- if (ls != null) {
- ctx.localizePaths(ls);
- }
- }
-
/**
* Clear the Fetch Operator.
*
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Tue Jul 9 09:07:35 2013
@@ -1105,6 +1105,10 @@ public final class FunctionRegistry {
GenericUDFBridge bridge = (GenericUDFBridge) genericUDF;
return new GenericUDFBridge(bridge.getUdfName(), bridge.isOperator(),
bridge.getUdfClass());
+ } else if (genericUDF instanceof GenericUDFMacro) {
+ GenericUDFMacro bridge = (GenericUDFMacro) genericUDF;
+ return new GenericUDFMacro(bridge.getMacroName(), bridge.getBody(),
+ bridge.getColNames(), bridge.getColTypes());
}
return (GenericUDF) ReflectionUtils
@@ -1173,6 +1177,11 @@ public final class FunctionRegistry {
}
}
+ if (genericUDF instanceof GenericUDFMacro) {
+ GenericUDFMacro macro = (GenericUDFMacro) (genericUDF);
+ return macro.isDeterministic();
+ }
+
return true;
}
@@ -1193,6 +1202,11 @@ public final class FunctionRegistry {
}
}
+ if (genericUDF instanceof GenericUDFMacro) {
+ GenericUDFMacro macro = (GenericUDFMacro) (genericUDF);
+ return macro.isStateful();
+ }
+
return false;
}
@@ -1303,6 +1317,27 @@ public final class FunctionRegistry {
}
/**
+ * Registers thae appropriate kind of temporary function based on a class's
+ * type.
+ *
+ * @param macroName name under which to register the macro
+ *
+ * @param body the expression which the macro evaluates to
+ *
+ * @param colNames the names of the arguments to the macro
+ *
+ * @param colTypes the types of the arguments to the macro
+ */
+ public static void registerTemporaryMacro(
+ String macroName, ExprNodeDesc body,
+ List<String> colNames, List<TypeInfo> colTypes) {
+
+ FunctionInfo fI = new FunctionInfo(false, macroName,
+ new GenericUDFMacro(macroName, body, colNames, colTypes));
+ mFunctions.put(macroName.toLowerCase(), fI);
+ }
+
+ /**
* Registers Hive functions from a plugin jar, using metadata from
* the jar's META-INF/class-info.xml.
*
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java Tue Jul 9 09:07:35 2013
@@ -28,6 +28,8 @@ import org.apache.hadoop.hive.ql.QueryPl
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.CreateFunctionDesc;
import org.apache.hadoop.hive.ql.plan.DropFunctionDesc;
+import org.apache.hadoop.hive.ql.plan.CreateMacroDesc;
+import org.apache.hadoop.hive.ql.plan.DropMacroDesc;
import org.apache.hadoop.hive.ql.plan.FunctionWork;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver;
@@ -67,6 +69,16 @@ public class FunctionTask extends Task<F
if (dropFunctionDesc != null) {
return dropFunction(dropFunctionDesc);
}
+
+ CreateMacroDesc createMacroDesc = work.getCreateMacroDesc();
+ if (createMacroDesc != null) {
+ return createMacro(createMacroDesc);
+ }
+
+ DropMacroDesc dropMacroDesc = work.getDropMacroDesc();
+ if (dropMacroDesc != null) {
+ return dropMacro(dropMacroDesc);
+ }
return 0;
}
@@ -89,6 +101,26 @@ public class FunctionTask extends Task<F
}
}
+ private int createMacro(CreateMacroDesc createMacroDesc) {
+ FunctionRegistry.registerTemporaryMacro(
+ createMacroDesc.getMacroName(),
+ createMacroDesc.getBody(),
+ createMacroDesc.getColNames(),
+ createMacroDesc.getColTypes());
+ return 0;
+ }
+
+ private int dropMacro(DropMacroDesc dropMacroDesc) {
+ try {
+ FunctionRegistry.unregisterTemporaryUDF(dropMacroDesc
+ .getMacroName());
+ return 0;
+ } catch (HiveException e) {
+ LOG.info("drop macro: " + StringUtils.stringifyException(e));
+ return 1;
+ }
+ }
+
private int dropFunction(DropFunctionDesc dropFunctionDesc) {
try {
FunctionRegistry.unregisterTemporaryUDF(dropFunctionDesc
@@ -114,9 +146,4 @@ public class FunctionTask extends Task<F
public String getName() {
return "FUNCTION";
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- throw new RuntimeException ("Unexpected call");
- }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java Tue Jul 9 09:07:35 2013
@@ -93,8 +93,9 @@ public class JoinOperator extends Common
.toString());
List keyObject = (List) soi.getStructFieldData(row, sf);
// Are we consuming too much memory
- if (alias == numAliases - 1 && !(handleSkewJoin && skewJoinKeyContext.currBigKeyTag >= 0)) {
- if (sz == joinEmitInterval) {
+ if (alias == numAliases - 1 && !(handleSkewJoin && skewJoinKeyContext.currBigKeyTag >= 0) &&
+ !hasLeftSemiJoin) {
+ if (sz == joinEmitInterval && !hasFilter(alias)) {
// The input is sorted by alias, so if we are already in the last join
// operand,
// we can emit some results now.
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Tue Jul 9 09:07:35 2013
@@ -242,10 +242,12 @@ public class MapOperator extends Operato
SerDeException {
PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile);
LinkedHashMap<String, String> partSpec = pd.getPartSpec();
- // Use tblProps in case of unpartitioned tables
+ // Use table properties in case of unpartitioned tables,
+ // and the union of table properties and partition properties, with partition
+ // taking precedence
Properties partProps =
(pd.getPartSpec() == null || pd.getPartSpec().isEmpty()) ?
- pd.getTableDesc().getProperties() : pd.getProperties();
+ pd.getTableDesc().getProperties() : pd.getOverlayedProperties();
Class serdeclass = pd.getDeserializerClass();
if (serdeclass == null) {
@@ -409,7 +411,7 @@ public class MapOperator extends Operato
// If the partition does not exist, use table properties
Properties partProps =
(pd.getPartSpec() == null || pd.getPartSpec().isEmpty()) ?
- tblProps : pd.getProperties();
+ tblProps : pd.getOverlayedProperties();
Class sdclass = pd.getDeserializerClass();
if (sdclass == null) {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java Tue Jul 9 09:07:35 2013
@@ -448,11 +448,6 @@ public class MapredLocalTask extends Tas
}
@Override
- public void localizeMRTmpFilesImpl(Context ctx) {
-
- }
-
- @Override
public boolean isMapRedLocalTask() {
return true;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java Tue Jul 9 09:07:35 2013
@@ -537,10 +537,4 @@ public class MoveTask extends Task<MoveW
public String getName() {
return "MOVE";
}
-
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // no-op
- }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java Tue Jul 9 09:07:35 2013
@@ -1,4 +1,9 @@
/**
+ <<<<<<< HEAD
+ =======
+ * Copyright 2010 The Apache Software Foundation
+ *
+ >>>>>>> HIVE-1402 [jira] Add parallel ORDER BY to Hive
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -28,6 +33,11 @@ public class OperatorUtils {
return findOperator(start, clazz, new HashSet<T>());
}
+ public static <T> T findSingleOperator(Operator<?> start, Class<T> clazz) {
+ Set<T> found = findOperator(start, clazz, new HashSet<T>());
+ return found.size() == 1 ? found.iterator().next() : null;
+ }
+
public static <T> Set<T> findOperators(Collection<Operator<?>> starts, Class<T> clazz) {
Set<T> found = new HashSet<T>();
for (Operator<?> start : starts) {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java Tue Jul 9 09:07:35 2013
@@ -107,6 +107,7 @@ public class PTFOperator extends Operato
processInputPartition();
}
}
+ inputPart.close();
}
@Override
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java Tue Jul 9 09:07:35 2013
@@ -152,6 +152,10 @@ public class PTFPartition
return new PItr(start, end);
}
+ public void close() {
+ elems.close();
+ }
+
class PItr implements PTFPartitionIterator<Object>
{
int idx;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPersistence.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPersistence.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPersistence.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPersistence.java Tue Jul 9 09:07:35 2013
@@ -37,6 +37,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -287,6 +289,11 @@ public class PTFPersistence {
bldr.append("]\n");
}
+ public void close() {
+ bytes = null;
+ offsetsArray = null;
+ }
+
class WIterator implements Iterator<Writable>
{
Writable wObj;
@@ -403,6 +410,12 @@ public class PTFPersistence {
public static class PartitionedByteBasedList extends ByteBasedList
{
+ private static final ShutdownHook hook = new ShutdownHook();
+
+ static {
+ Runtime.getRuntime().addShutdownHook(hook);
+ }
+
ArrayList<ByteBasedList> partitions;
ArrayList<Integer> partitionOffsets;
ArrayList<File> reusableFiles;
@@ -413,8 +426,7 @@ public class PTFPersistence {
{
this.batchSize = batchSize;
currentSize = 0;
- dir = PartitionedByteBasedList.createTempDir();
- Runtime.getRuntime().addShutdownHook(new ShutdownHook(dir));
+ hook.register(dir = PartitionedByteBasedList.createTempDir());
partitions = new ArrayList<ByteBasedList>();
partitionOffsets = new ArrayList<Integer>();
@@ -446,6 +458,22 @@ public class PTFPersistence {
}
}
+ @Override
+ public void close() {
+ super.close();
+ reusableFiles.clear();
+ partitionOffsets.clear();
+ for (ByteBasedList partition : partitions) {
+ partition.close();
+ }
+ partitions.clear();
+ try {
+ PartitionedByteBasedList.deleteRecursively(dir);
+ } catch (Exception e) {
+ }
+ hook.unregister(dir);
+ }
+
private void addPartition() throws HiveException
{
try
@@ -636,11 +664,14 @@ public class PTFPersistence {
static class ShutdownHook extends Thread
{
- File dir;
+ private final Set<File> dirs = new LinkedHashSet<File>();
- public ShutdownHook(File dir)
- {
- this.dir = dir;
+ public void register(File dir) {
+ dirs.add(dir);
+ }
+
+ public void unregister(File dir) {
+ dirs.remove(dir);
}
@Override
@@ -648,9 +679,11 @@ public class PTFPersistence {
{
try
{
- PartitionedByteBasedList.deleteRecursively(dir);
+ for (File dir : dirs) {
+ PartitionedByteBasedList.deleteRecursively(dir);
+ }
}
- catch(IOException ie)
+ catch(Exception ie)
{
}
}
@@ -852,6 +885,21 @@ public class PTFPersistence {
throw new HiveException("Reset on PersistentByteBasedList not supported");
}
+ @Override
+ public void close() {
+ super.close();
+ ByteBasedList list = memList.get();
+ if (list != null) {
+ list.close();
+ }
+ memList.clear();
+ try {
+ PartitionedByteBasedList.deleteRecursively(file);
+ } catch (Exception e) {
+ // ignore
+ }
+ }
+
private ByteBasedList getList() throws HiveException
{
PTFPersistence.lock(lock.readLock());
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java Tue Jul 9 09:07:35 2013
@@ -263,11 +263,6 @@ public class StatsTask extends Task<Stat
return "STATS";
}
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // Nothing to do for StatsTask here.
- }
-
private int aggregateStats() {
StatsAggregator statsAggregator = null;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java Tue Jul 9 09:07:35 2013
@@ -95,6 +95,12 @@ public abstract class Task<T extends Ser
protected List<Task<? extends Serializable>> childTasks;
protected List<Task<? extends Serializable>> parentTasks;
+ /**
+ * this can be set by the Task, to provide more info about the failure in TaskResult
+ * where the Driver can find it. This is checked if {@link Task#execute(org.apache.hadoop.hive.ql.DriverContext)}
+ * returns non-0 code.
+ */
+ private Throwable exception;
public Task() {
isdone = false;
@@ -384,37 +390,6 @@ public abstract class Task<T extends Ser
public abstract StageType getType();
/**
- * If this task uses any map-reduce intermediate data (either for reading or for writing),
- * localize them (using the supplied Context). Map-Reduce intermediate directories are allocated
- * using Context.getMRTmpFileURI() and can be localized using localizeMRTmpFileURI().
- *
- * This method is declared abstract to force any task code to explicitly deal with this aspect of
- * execution.
- *
- * @param ctx
- * context object with which to localize
- */
- abstract protected void localizeMRTmpFilesImpl(Context ctx);
-
- /**
- * Localize a task tree
- *
- * @param ctx
- * context object with which to localize
- */
- public final void localizeMRTmpFiles(Context ctx) {
- localizeMRTmpFilesImpl(ctx);
-
- if (childTasks == null) {
- return;
- }
-
- for (Task<? extends Serializable> t : childTasks) {
- t.localizeMRTmpFiles(ctx);
- }
- }
-
- /**
* Subscribe the feed of publisher. To prevent cycles, a task can only subscribe to its ancestor.
* Feed is a generic form of execution-time feedback (type, value) pair from one task to another
* task. Examples include dynamic partitions (which are only available at execution time). The
@@ -525,4 +500,10 @@ public abstract class Task<T extends Ser
public List<FieldSchema> getResultSchema() {
return null;
}
+ Throwable getException() {
+ return exception;
+ }
+ void setException(Throwable ex) {
+ exception = ex;
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskResult.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskResult.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskResult.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskResult.java Tue Jul 9 09:07:35 2013
@@ -20,11 +20,13 @@ package org.apache.hadoop.hive.ql.exec;
/**
* TaskResult implementation.
+ * Note that different threads may be reading/writing this object
**/
public class TaskResult {
- protected int exitVal;
- protected boolean runStatus;
+ protected volatile int exitVal;
+ protected volatile boolean runStatus;
+ private volatile Throwable taskError;
public TaskResult() {
exitVal = -1;
@@ -35,11 +37,21 @@ public class TaskResult {
this.exitVal = exitVal;
setRunning(false);
}
+ public void setExitVal(int exitVal, Throwable taskError) {
+ this.setExitVal(exitVal);
+ this.taskError = taskError;
+ }
public int getExitVal() {
return exitVal;
}
+ /**
+ * @return may contain details of the error which caused the task to fail or null
+ */
+ public Throwable getTaskError() {
+ return taskError;
+ }
public boolean isRunning() {
return runStatus;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java Tue Jul 9 09:07:35 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -30,6 +31,13 @@ public class TaskRunner extends Thread {
protected Task<? extends Serializable> tsk;
protected TaskResult result;
protected SessionState ss;
+ private static AtomicLong taskCounter = new AtomicLong(0);
+ private static ThreadLocal<Long> taskRunnerID = new ThreadLocal<Long>() {
+ @Override
+ protected Long initialValue() {
+ return taskCounter.incrementAndGet();
+ }
+ };
public TaskRunner(Task<? extends Serializable> tsk, TaskResult result) {
this.tsk = tsk;
@@ -58,7 +66,10 @@ public class TaskRunner extends Thread {
} catch (Throwable t) {
t.printStackTrace();
}
- result.setExitVal(exitVal);
+ result.setExitVal(exitVal, tsk.getException());
}
+ public static long getTaskRunnerID () {
+ return taskRunnerID.get();
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java Tue Jul 9 09:07:35 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
+import java.util.Arrays;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -28,6 +29,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.UDTFDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.ql.udf.generic.UDTFCollector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
@@ -45,7 +47,10 @@ public class UDTFOperator extends Operat
ObjectInspector[] udtfInputOIs = null;
Object[] objToSendToUDTF = null;
- Object[] forwardObj = new Object[1];
+
+ GenericUDTF genericUDTF;
+ UDTFCollector collector;
+ List outerObj;
/**
* sends periodic reports back to the tracker.
@@ -54,7 +59,10 @@ public class UDTFOperator extends Operat
@Override
protected void initializeOp(Configuration hconf) throws HiveException {
- conf.getGenericUDTF().setCollector(new UDTFCollector(this));
+ genericUDTF = conf.getGenericUDTF();
+ collector = new UDTFCollector(this);
+
+ genericUDTF.setCollector(collector);
// Make an object inspector [] of the arguments to the UDTF
List<? extends StructField> inputFields =
@@ -68,10 +76,13 @@ public class UDTFOperator extends Operat
MapredContext context = MapredContext.get();
if (context != null) {
- context.setup(conf.getGenericUDTF());
+ context.setup(genericUDTF);
}
- StructObjectInspector udtfOutputOI = conf.getGenericUDTF().initialize(
+ StructObjectInspector udtfOutputOI = genericUDTF.initialize(
udtfInputOIs);
+ if (conf.isOuterLV()) {
+ outerObj = Arrays.asList(new Object[udtfOutputOI.getAllStructFieldRefs().size()]);
+ }
// Since we're passing the object output by the UDTF directly to the next
// operator, we can use the same OI.
@@ -100,8 +111,11 @@ public class UDTFOperator extends Operat
objToSendToUDTF[i] = soi.getStructFieldData(row, fields.get(i));
}
- conf.getGenericUDTF().process(objToSendToUDTF);
-
+ genericUDTF.process(objToSendToUDTF);
+ if (conf.isOuterLV() && collector.getCounter() == 0) {
+ collector.collect(outerObj);
+ }
+ collector.reset();
}
/**
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Tue Jul 9 09:07:35 2013
@@ -1313,6 +1313,18 @@ public final class Utilities {
}
/**
+ * returns null if path is not exist
+ */
+ public static FileStatus[] listStatusIfExists(Path path, FileSystem fs) throws IOException {
+ try {
+ return fs.listStatus(path);
+ } catch (FileNotFoundException e) {
+ // FS in hadoop 2.0 throws FNF instead of returning null
+ return null;
+ }
+ }
+
+ /**
* Get all file status from a root path and recursively go deep into certain levels.
*
* @param path
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/DataCorruptErrorHeuristic.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/DataCorruptErrorHeuristic.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/DataCorruptErrorHeuristic.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/DataCorruptErrorHeuristic.java Tue Jul 9 09:07:35 2013
@@ -36,7 +36,7 @@ import java.util.regex.Pattern;
public class DataCorruptErrorHeuristic extends RegexErrorHeuristic {
- private static final String SPLIT_REGEX = "split:.*";
+ private static final String SPLIT_REGEX = "split:\\s*([^\\s]+)";
private static final String EXCEPTION_REGEX = "EOFException";
public DataCorruptErrorHeuristic() {
@@ -55,14 +55,13 @@ public class DataCorruptErrorHeuristic e
rll.get(SPLIT_REGEX).size() > 0) {
// There should only be a single split line...
- assert(rll.get(SPLIT_REGEX).size()==1);
String splitLogLine = rll.get(SPLIT_REGEX).get(0);
// Extract only 'split: hdfs://...'
Pattern p = Pattern.compile(SPLIT_REGEX, Pattern.CASE_INSENSITIVE);
Matcher m = p.matcher(splitLogLine);
m.find();
- String splitStr = m.group();
+ String splitStr = m.group(1);
es = new ErrorAndSolution(
"Data file " + splitStr + " is corrupted.",
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/ErrorAndSolution.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/ErrorAndSolution.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/ErrorAndSolution.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/ErrorAndSolution.java Tue Jul 9 09:07:35 2013
@@ -23,8 +23,8 @@ package org.apache.hadoop.hive.ql.exec.e
*/
public class ErrorAndSolution {
- private String error = null;
- private String solution = null;
+ private final String error;
+ private final String solution;
ErrorAndSolution(String error, String solution) {
this.error = error;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/RegexErrorHeuristic.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/RegexErrorHeuristic.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/RegexErrorHeuristic.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/RegexErrorHeuristic.java Tue Jul 9 09:07:35 2013
@@ -40,7 +40,6 @@ import org.apache.hadoop.mapred.JobConf;
*/
public abstract class RegexErrorHeuristic implements ErrorHeuristic {
- private String query = null;
private JobConf conf = null;
// Pattern to look for in the hive query and whether it matched
@@ -86,10 +85,9 @@ public abstract class RegexErrorHeuristi
* Before init is called, logRegexes and queryRegexes should be populated.
*/
public void init(String query, JobConf conf) {
- this.query = query;
this.conf = conf;
- assert((logRegexes!=null) && (queryRegex != null));
+ assert(queryRegex != null);
Pattern queryPattern = Pattern.compile(queryRegex, Pattern.CASE_INSENSITIVE);
queryMatches = queryPattern.matcher(query).find();
@@ -98,7 +96,6 @@ public abstract class RegexErrorHeuristi
regexToPattern.put(regex, Pattern.compile(regex, Pattern.CASE_INSENSITIVE));
regexToLogLines.put(regex, new ArrayList<String>());
}
-
}
@Override
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java Tue Jul 9 09:07:35 2013
@@ -47,13 +47,11 @@ public class TaskLogProcessor {
new HashMap<ErrorHeuristic, HeuristicStats>();
private final List<String> taskLogUrls = new ArrayList<String>();
- private JobConf conf = null;
// Query is the hive query string i.e. "SELECT * FROM src;" associated with
// this set of tasks logs
- private String query = null;
+ private final String query;
public TaskLogProcessor(JobConf conf) {
- this.conf = conf;
query = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING);
heuristics.put(new ScriptErrorHeuristic(), new HeuristicStats());
@@ -197,7 +195,6 @@ public class TaskLogProcessor {
try {
in = new BufferedReader(
new InputStreamReader(taskAttemptLogUrl.openStream()));
- String inputLine;
String lastLine = null;
boolean lastLineMatched = false;
List<String> stackTrace = null;
@@ -207,9 +204,20 @@ public class TaskLogProcessor {
Pattern endStackTracePattern =
Pattern.compile("^\t... [0-9]+ more.*", Pattern.CASE_INSENSITIVE);
- while ((inputLine =
- ShimLoader.getHadoopShims().unquoteHtmlChars(in.readLine())) != null) {
-
+ String inputLine;
+ while (true) {
+ inputLine = in.readLine();
+ if (inputLine == null) { // EOF:
+ if (stackTrace != null) {
+ stackTraces.add(stackTrace);
+ stackTrace = null;
+ }
+ break;
+ }
+
+ inputLine =
+ ShimLoader.getHadoopShims().unquoteHtmlChars(inputLine);
+
if (stackTracePattern.matcher(inputLine).matches() ||
endStackTracePattern.matcher(inputLine).matches()) {
// We are in a stack trace
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java Tue Jul 9 09:07:35 2013
@@ -287,7 +287,7 @@ public class HiveHistory {
String key = ent.getKey();
String val = ent.getValue();
if(val != null) {
- val = val.replace('\n', ' ');
+ val = val.replace(System.getProperty("line.separator"), " ");
}
sb.append(key + "=\"" + val + "\"");
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java Tue Jul 9 09:07:35 2013
@@ -98,9 +98,4 @@ public class IndexMetadataChangeTask ext
public StageType getType() {
return StageType.DDL;
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- }
-
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/NonSyncDataInputBuffer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/NonSyncDataInputBuffer.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/NonSyncDataInputBuffer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/NonSyncDataInputBuffer.java Tue Jul 9 09:07:35 2013
@@ -118,6 +118,7 @@ public class NonSyncDataInputBuffer exte
* If a problem occurs reading from this DataInputStream.
*
*/
+ @Override
public final boolean readBoolean() throws IOException {
int temp = in.read();
if (temp < 0) {
@@ -135,6 +136,7 @@ public class NonSyncDataInputBuffer exte
* If a problem occurs reading from this DataInputStream.
*
*/
+ @Override
public final byte readByte() throws IOException {
int temp = in.read();
if (temp < 0) {
@@ -165,6 +167,7 @@ public class NonSyncDataInputBuffer exte
return offset;
}
+ @Override
public final char readChar() throws IOException {
if (readToBuff(2) < 0) {
throw new EOFException();
@@ -182,6 +185,7 @@ public class NonSyncDataInputBuffer exte
* If a problem occurs reading from this DataInputStream.
*
*/
+ @Override
public final double readDouble() throws IOException {
return Double.longBitsToDouble(readLong());
}
@@ -195,6 +199,7 @@ public class NonSyncDataInputBuffer exte
* If a problem occurs reading from this DataInputStream.
*
*/
+ @Override
public final float readFloat() throws IOException {
return Float.intBitsToFloat(readInt());
}
@@ -211,6 +216,7 @@ public class NonSyncDataInputBuffer exte
* If a problem occurs reading from this DataInputStream.
*
*/
+ @Override
public final void readFully(byte[] buffer) throws IOException {
readFully(buffer, 0, buffer.length);
}
@@ -233,6 +239,7 @@ public class NonSyncDataInputBuffer exte
* if reaches the end of the stream before enough bytes have been
* read
*/
+ @Override
public final void readFully(byte[] buffer, int offset, int length)
throws IOException {
if (length < 0) {
@@ -267,6 +274,7 @@ public class NonSyncDataInputBuffer exte
* If a problem occurs reading from this DataInputStream.
*
*/
+ @Override
public final int readInt() throws IOException {
if (readToBuff(4) < 0) {
throw new EOFException();
@@ -292,6 +300,7 @@ public class NonSyncDataInputBuffer exte
* @deprecated Use BufferedReader
*/
@Deprecated
+ @Override
public final String readLine() throws IOException {
StringBuilder line = new StringBuilder(80); // Typical line length
boolean foundTerminator = false;
@@ -335,6 +344,7 @@ public class NonSyncDataInputBuffer exte
* If a problem occurs reading from this DataInputStream.
*
*/
+ @Override
public final long readLong() throws IOException {
if (readToBuff(8) < 0) {
throw new EOFException();
@@ -356,6 +366,7 @@ public class NonSyncDataInputBuffer exte
* If a problem occurs reading from this DataInputStream.
*
*/
+ @Override
public final short readShort() throws IOException {
if (readToBuff(2) < 0) {
throw new EOFException();
@@ -373,6 +384,7 @@ public class NonSyncDataInputBuffer exte
* If a problem occurs reading from this DataInputStream.
*
*/
+ @Override
public final int readUnsignedByte() throws IOException {
int temp = in.read();
if (temp < 0) {
@@ -391,6 +403,7 @@ public class NonSyncDataInputBuffer exte
* If a problem occurs reading from this DataInputStream.
*
*/
+ @Override
public final int readUnsignedShort() throws IOException {
if (readToBuff(2) < 0) {
throw new EOFException();
@@ -407,6 +420,7 @@ public class NonSyncDataInputBuffer exte
* If a problem occurs reading from this DataInputStream.
*
*/
+ @Override
public final String readUTF() throws IOException {
return decodeUTF(readUnsignedShort());
}
@@ -450,6 +464,7 @@ public class NonSyncDataInputBuffer exte
* @throws IOException
* If the stream is already closed or another IOException occurs.
*/
+ @Override
public final int skipBytes(int count) throws IOException {
int skipped = 0;
long skip;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java Tue Jul 9 09:07:35 2013
@@ -34,6 +34,8 @@ class OutStream extends PositionedOutput
static final int HEADER_SIZE = 3;
private final String name;
private final OutputReceiver receiver;
+ // if enabled the stream will be suppressed when writing stripe
+ private boolean suppress;
/**
* Stores the uncompressed bytes that have been serialized, but not
@@ -70,6 +72,7 @@ class OutStream extends PositionedOutput
this.bufferSize = bufferSize;
this.codec = codec;
this.receiver = receiver;
+ this.suppress = false;
}
public void clear() throws IOException {
@@ -78,6 +81,7 @@ class OutStream extends PositionedOutput
compressed = null;
overflow = null;
current = null;
+ suppress = false;
}
/**
@@ -264,5 +268,20 @@ class OutStream extends PositionedOutput
}
return result;
}
+
+ /**
+ * Set suppress flag
+ */
+ public void suppress() {
+ suppress = true;
+ }
+
+ /**
+ * Returns the state of suppress flag
+ * @return value of suppress flag
+ */
+ public boolean isSuppressed() {
+ return suppress;
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java Tue Jul 9 09:07:35 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -231,6 +232,14 @@ class WriterImpl implements Writer, Memo
}
/**
+ * Check the state of suppress flag in output stream
+ * @return value of suppress flag
+ */
+ public boolean isSuppressed() {
+ return outStream.isSuppressed();
+ }
+
+ /**
* Write the saved compressed buffers to the OutputStream.
* @param out the stream to write to
* @throws IOException
@@ -291,9 +300,9 @@ class WriterImpl implements Writer, Memo
* @return The output outStream that the section needs to be written to.
* @throws IOException
*/
- public PositionedOutputStream createStream(int column,
- OrcProto.Stream.Kind kind
- ) throws IOException {
+ public OutStream createStream(int column,
+ OrcProto.Stream.Kind kind
+ ) throws IOException {
StreamName name = new StreamName(column, kind);
BufferedStream result = streams.get(name);
if (result == null) {
@@ -325,6 +334,14 @@ class WriterImpl implements Writer, Memo
public boolean buildIndex() {
return buildIndex;
}
+
+ /**
+ * Is the ORC file compressed?
+ * @return are the streams compressed
+ */
+ public boolean isCompressed() {
+ return codec != null;
+ }
}
/**
@@ -337,6 +354,7 @@ class WriterImpl implements Writer, Memo
protected final int id;
protected final ObjectInspector inspector;
private final BitFieldWriter isPresent;
+ private final boolean isCompressed;
protected final ColumnStatisticsImpl indexStatistics;
private final ColumnStatisticsImpl fileStatistics;
protected TreeWriter[] childrenWriters;
@@ -344,6 +362,8 @@ class WriterImpl implements Writer, Memo
private final OrcProto.RowIndex.Builder rowIndex;
private final OrcProto.RowIndexEntry.Builder rowIndexEntry;
private final PositionedOutputStream rowIndexStream;
+ private boolean foundNulls;
+ private OutStream isPresentOutStream;
/**
* Create a tree writer.
@@ -356,14 +376,17 @@ class WriterImpl implements Writer, Memo
TreeWriter(int columnId, ObjectInspector inspector,
StreamFactory streamFactory,
boolean nullable) throws IOException {
+ this.isCompressed = streamFactory.isCompressed();
this.id = columnId;
this.inspector = inspector;
if (nullable) {
- isPresent = new BitFieldWriter(streamFactory.createStream(id,
- OrcProto.Stream.Kind.PRESENT), 1);
+ isPresentOutStream = streamFactory.createStream(id,
+ OrcProto.Stream.Kind.PRESENT);
+ isPresent = new BitFieldWriter(isPresentOutStream, 1);
} else {
isPresent = null;
}
+ this.foundNulls = false;
indexStatistics = ColumnStatisticsImpl.create(inspector);
fileStatistics = ColumnStatisticsImpl.create(inspector);
childrenWriters = new TreeWriter[0];
@@ -401,6 +424,20 @@ class WriterImpl implements Writer, Memo
}
if (isPresent != null) {
isPresent.write(obj == null ? 0 : 1);
+ if(obj == null) {
+ foundNulls = true;
+ }
+ }
+ }
+
+ private void removeIsPresentPositions() {
+ for(int i=0; i < rowIndex.getEntryCount(); ++i) {
+ RowIndexEntry.Builder entry = rowIndex.getEntryBuilder(i);
+ List<Long> positions = entry.getPositionsList();
+ // bit streams use 3 positions if uncompressed, 4 if compressed
+ positions = positions.subList(isCompressed ? 4 : 3, positions.size());
+ entry.clearPositions();
+ entry.addAllPositions(positions);
}
}
@@ -418,7 +455,21 @@ class WriterImpl implements Writer, Memo
int requiredIndexEntries) throws IOException {
if (isPresent != null) {
isPresent.flush();
+
+ // if no nulls are found in a stream, then suppress the stream
+ if(!foundNulls) {
+ isPresentOutStream.suppress();
+ // since isPresent bitstream is suppressed, update the index to
+ // remove the positions of the isPresent stream
+ if (rowIndexStream != null) {
+ removeIsPresentPositions();
+ }
+ }
}
+
+ // reset the flag for next stripe
+ foundNulls = false;
+
builder.addColumns(getEncoding());
if (rowIndexStream != null) {
if (rowIndex.getEntryCount() != requiredIndexEntries) {
@@ -810,6 +861,7 @@ class WriterImpl implements Writer, Memo
* and augments them with the final information as the stripe is written.
* @throws IOException
*/
+ @Override
void createRowIndexEntry() throws IOException {
getFileStatistics().merge(indexStatistics);
OrcProto.RowIndexEntry.Builder rowIndexEntry = getRowIndexEntry();
@@ -1349,19 +1401,21 @@ class WriterImpl implements Writer, Memo
long indexEnd = start;
for(Map.Entry<StreamName, BufferedStream> pair: streams.entrySet()) {
BufferedStream stream = pair.getValue();
- stream.flush();
- stream.spillTo(rawWriter);
- stream.clear();
- long end = rawWriter.getPos();
- StreamName name = pair.getKey();
- builder.addStreams(OrcProto.Stream.newBuilder()
- .setColumn(name.getColumn())
- .setKind(name.getKind())
- .setLength(end-section));
- section = end;
- if (StreamName.Area.INDEX == name.getArea()) {
- indexEnd = end;
+ if (!stream.isSuppressed()) {
+ stream.flush();
+ stream.spillTo(rawWriter);
+ long end = rawWriter.getPos();
+ StreamName name = pair.getKey();
+ builder.addStreams(OrcProto.Stream.newBuilder()
+ .setColumn(name.getColumn())
+ .setKind(name.getKind())
+ .setLength(end-section));
+ section = end;
+ if (StreamName.Area.INDEX == name.getArea()) {
+ indexEnd = end;
+ }
}
+ stream.clear();
}
builder.build().writeTo(protobufWriter);
protobufWriter.flush();
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java Tue Jul 9 09:07:35 2013
@@ -378,10 +378,4 @@ public class BlockMergeTask extends Task
public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
// no op
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // no op
- }
-
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java Tue Jul 9 09:07:35 2013
@@ -379,11 +379,4 @@ public class PartialScanTask extends Tas
public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
// no op
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // no op
- }
-
-
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java Tue Jul 9 09:07:35 2013
@@ -254,10 +254,4 @@ public class ColumnTruncateTask extends
public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
// no op
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // no op
- }
-
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java Tue Jul 9 09:07:35 2013
@@ -60,6 +60,7 @@ public class DefaultRuleDispatcher imple
* the operators encountered so far
* @throws SemanticException
*/
+ @Override
public Object dispatch(Node nd, Stack<Node> ndStack, Object... nodeOutputs)
throws SemanticException {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java Tue Jul 9 09:07:35 2013
@@ -58,9 +58,10 @@ public class RuleRegExp implements Rule
* @return cost of the function
* @throws SemanticException
*/
+ @Override
public int cost(Stack<Node> stack) throws SemanticException {
int numElems = (stack != null ? stack.size() : 0);
- String name = new String();
+ String name = "";
for (int pos = numElems - 1; pos >= 0; pos--) {
name = stack.get(pos).getName() + "%" + name;
Matcher m = pattern.matcher(name);
@@ -68,13 +69,13 @@ public class RuleRegExp implements Rule
return m.group().length();
}
}
-
return -1;
}
/**
* @return the name of the Node
**/
+ @Override
public String getName() {
return ruleName;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java Tue Jul 9 09:07:35 2013
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.lockmgr;
+import java.util.Arrays;
+
import org.apache.hadoop.hive.ql.metadata.DummyPartition;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -94,16 +96,16 @@ public class HiveLockObject {
}
HiveLockObjectData target = (HiveLockObjectData) o;
- boolean ret = (queryId == null) ? target.getQueryId() == null :
- queryId.equals(target.getQueryId());
- ret = ret && (lockTime == null) ? target.getLockTime() == null :
- queryId.equals(target.getLockTime());
- ret = ret && (lockMode == null) ? target.getLockMode() == null :
- queryId.equals(target.getLockMode());
- ret = ret && (queryStr == null) ? target.getQueryStr() == null :
- queryStr.equals(target.getQueryStr());
- ret = ret && (clientIp == null) ? target.getClientIp() == null :
- clientIp.equals(target.getClientIp());
+ boolean ret = (queryId == null ? target.queryId == null :
+ target.queryId != null && queryId.equals(target.queryId));
+ ret = ret && (lockTime == null ? target.lockTime == null :
+ target.lockTime != null && lockTime.equals(target.lockTime));
+ ret = ret && (lockMode == null ? target.lockMode == null :
+ target.lockMode != null && lockMode.equals(target.lockMode));
+ ret = ret && (queryStr == null ? target.queryStr == null :
+ target.queryStr != null && queryStr.equals(target.queryStr));
+ ret = ret && (clientIp == null ? target.clientIp == null :
+ target.clientIp != null && clientIp.equals(target.clientIp));
return ret;
}
@@ -145,20 +147,17 @@ public class HiveLockObject {
}
public String getName() {
- if (this.pathNames == null) {
+ if (pathNames == null) {
return null;
}
- String ret = "";
- boolean first = true;
+ StringBuilder builder = new StringBuilder();
for (int i = 0; i < pathNames.length; i++) {
- if (!first) {
- ret = ret + "/";
- } else {
- first = false;
+ if (i > 0) {
+ builder.append('/');
}
- ret = ret + pathNames[i];
+ builder.append(pathNames[i]);
}
- return ret;
+ return builder.toString();
}
public String getDisplayName() {
@@ -200,7 +199,8 @@ public class HiveLockObject {
}
HiveLockObject tgt = (HiveLockObject) o;
- return getName().equals(tgt.getName()) &&
- data == null ? tgt.getData() == null : data.equals(tgt.getData());
+ return Arrays.equals(pathNames, tgt.pathNames) &&
+ data == null ? tgt.getData() == null :
+ tgt.getData() != null && data.equals(tgt.getData());
}
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java Tue Jul 9 09:07:35 2013
@@ -55,6 +55,8 @@ import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.ZooKeeper;
+import com.google.common.annotations.VisibleForTesting;
+
public class ZooKeeperHiveLockManager implements HiveLockManager {
HiveLockManagerCtx ctx;
public static final Log LOG = LogFactory.getLog("ZooKeeperHiveLockManager");
@@ -439,20 +441,25 @@ public class ZooKeeperHiveLockManager im
}
/* Remove the lock specified */
- private static void unlockPrimitive(HiveConf conf, ZooKeeper zkpClient,
+ @VisibleForTesting
+ static void unlockPrimitive(HiveConf conf, ZooKeeper zkpClient,
HiveLock hiveLock, String parent) throws LockException {
ZooKeeperHiveLock zLock = (ZooKeeperHiveLock)hiveLock;
try {
+ // can throw KeeperException.NoNodeException, which might mean something is wrong
zkpClient.delete(zLock.getPath(), -1);
// Delete the parent node if all the children have been deleted
HiveLockObject obj = zLock.getHiveLockObject();
String name = getLastObjectName(parent, obj);
- List<String> children = zkpClient.getChildren(name, false);
- if ((children == null) || (children.isEmpty()))
- {
- zkpClient.delete(name, -1);
+ try {
+ List<String> children = zkpClient.getChildren(name, false);
+ if (children == null || children.isEmpty()) {
+ zkpClient.delete(name, -1);
+ }
+ } catch (KeeperException.NoNodeException e) {
+ LOG.debug("Node " + name + " previously deleted when attempting to delete.");
}
} catch (Exception e) {
LOG.error("Failed to release ZooKeeper lock: ", e);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Tue Jul 9 09:07:35 2013
@@ -729,8 +729,9 @@ public class Hive {
List<Order> sortCols = new ArrayList<Order>();
storageDescriptor.setBucketCols(null);
int k = 0;
- for (int i = 0; i < storageDescriptor.getCols().size(); i++) {
- FieldSchema col = storageDescriptor.getCols().get(i);
+ Table metaBaseTbl = new Table(baseTbl);
+ for (int i = 0; i < metaBaseTbl.getCols().size(); i++) {
+ FieldSchema col = metaBaseTbl.getCols().get(i);
if (indexedCols.contains(col.getName())) {
indexTblCols.add(col);
sortCols.add(new Order(col.getName(), 1));
@@ -950,7 +951,7 @@ public class Hive {
} catch (NoSuchObjectException e) {
if (throwException) {
LOG.error(StringUtils.stringifyException(e));
- throw new InvalidTableException("Table " + tableName + " not found ", tableName);
+ throw new InvalidTableException(tableName);
}
return null;
} catch (Exception e) {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveException.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveException.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveException.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveException.java Tue Jul 9 09:07:35 2013
@@ -18,11 +18,17 @@
package org.apache.hadoop.hive.ql.metadata;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+
/**
* Generic exception class for Hive.
*/
public class HiveException extends Exception {
+ /**
+ * Standard predefined message with error code and possibly SQL State, etc.
+ */
+ private ErrorMsg canonicalErrorMsg = ErrorMsg.GENERIC_ERROR;
public HiveException() {
super();
}
@@ -38,4 +44,26 @@ public class HiveException extends Excep
public HiveException(String message, Throwable cause) {
super(message, cause);
}
+
+ public HiveException(ErrorMsg message, String... msgArgs) {
+ this(null, message, msgArgs);
+ }
+
+ /**
+ * This is the recommended constructor to use since it helps use
+ * canonical messages throughout.
+ * @param errorMsg Canonical error message
+ * @param msgArgs message arguments if message is parametrized; must be {@code null} is message takes no arguments
+ */
+ public HiveException(Throwable cause, ErrorMsg errorMsg, String... msgArgs) {
+ super(errorMsg.format(msgArgs), cause);
+ canonicalErrorMsg = errorMsg;
+
+ }
+ /**
+ * @return {@link ErrorMsg#GENERIC_ERROR} by default
+ */
+ public ErrorMsg getCanonicalErrorMsg() {
+ return canonicalErrorMsg;
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/InvalidTableException.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/InvalidTableException.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/InvalidTableException.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/InvalidTableException.java Tue Jul 9 09:07:35 2013
@@ -18,31 +18,17 @@
package org.apache.hadoop.hive.ql.metadata;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+
/**
* Generic exception class for Hive.
- *
*/
public class InvalidTableException extends HiveException {
String tableName;
public InvalidTableException(String tableName) {
- super();
- this.tableName = tableName;
- }
-
- public InvalidTableException(String message, String tableName) {
- super(message);
- this.tableName = tableName;
- }
-
- public InvalidTableException(Throwable cause, String tableName) {
- super(cause);
- this.tableName = tableName;
- }
-
- public InvalidTableException(String message, Throwable cause, String tableName) {
- super(message, cause);
+ super(ErrorMsg.INVALID_TABLE, tableName);
this.tableName = tableName;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java Tue Jul 9 09:07:35 2013
@@ -48,12 +48,12 @@ import org.codehaus.jackson.map.ObjectMa
* json.
*/
public class JsonMetaDataFormatter implements MetaDataFormatter {
- private static final Log LOG = LogFactory.getLog("hive.ql.exec.DDLTask");
+ private static final Log LOG = LogFactory.getLog(JsonMetaDataFormatter.class);
/**
* Convert the map to a JSON string.
*/
- public void asJson(OutputStream out, Map<String, Object> data)
+ private void asJson(OutputStream out, Map<String, Object> data)
throws HiveException
{
try {
@@ -66,70 +66,29 @@ public class JsonMetaDataFormatter imple
/**
* Write an error message.
*/
- public void error(OutputStream out, String msg, int errorCode)
+ @Override
+ public void error(OutputStream out, String msg, int errorCode, String sqlState)
throws HiveException
{
- asJson(out,
- MapBuilder.create()
- .put("error", msg)
- .put("errorCode", errorCode)
- .build());
+ error(out, msg, errorCode, sqlState, null);
}
-
- /**
- * Write a log warn message.
- */
- public void logWarn(OutputStream out, String msg, int errorCode)
- throws HiveException
- {
- LOG.warn(msg);
- error(out, msg, errorCode);
- }
-
- /**
- * Write a log info message.
- */
- public void logInfo(OutputStream out, String msg, int errorCode)
- throws HiveException
- {
- LOG.info(msg);
- error(out, msg, errorCode);
- }
-
- /**
- * Write a console error message.
- */
- public void consoleError(LogHelper console, String msg, int errorCode) {
- try {
- console.printError(msg);
- error(console.getOutStream(), msg, errorCode);
- } catch (HiveException e) {
- console.printError("unable to create json: " + e);
+ @Override
+ public void error(OutputStream out, String errorMessage, int errorCode, String sqlState, String errorDetail) throws HiveException {
+ MapBuilder mb = MapBuilder.create().put("error", errorMessage);
+ if(errorDetail != null) {
+ mb.put("errorDetail", errorDetail);
}
- }
-
- /**
- * Write a console error message.
- */
- public void consoleError(LogHelper console, String msg, String detail,
- int errorCode)
- {
- try {
- console.printError(msg, detail);
- asJson(console.getOutStream(),
- MapBuilder.create()
- .put("error", msg)
- .put("errorDetail", detail)
- .put("errorCode", errorCode)
- .build());
- } catch (HiveException e) {
- console.printError("unable to create json: " + e);
+ mb.put("errorCode", errorCode);
+ if(sqlState != null) {
+ mb.put("sqlState", sqlState);
}
+ asJson(out,mb.build());
}
/**
* Show a list of tables.
*/
+ @Override
public void showTables(DataOutputStream out, Set<String> tables)
throws HiveException
{
@@ -142,6 +101,7 @@ public class JsonMetaDataFormatter imple
/**
* Describe table.
*/
+ @Override
public void describeTable(DataOutputStream out,
String colPath, String tableName,
Table tbl, Partition part, List<FieldSchema> cols,
@@ -178,6 +138,7 @@ public class JsonMetaDataFormatter imple
.build();
}
+ @Override
public void showTableStatus(DataOutputStream out,
Hive db,
HiveConf conf,
@@ -366,6 +327,7 @@ public class JsonMetaDataFormatter imple
/**
* Show the table partitions.
*/
+ @Override
public void showTablePartitons(DataOutputStream out, List<String> parts)
throws HiveException
{
@@ -424,6 +386,7 @@ public class JsonMetaDataFormatter imple
/**
* Show a list of databases
*/
+ @Override
public void showDatabases(DataOutputStream out, List<String> databases)
throws HiveException
{
@@ -436,6 +399,7 @@ public class JsonMetaDataFormatter imple
/**
* Show the description of a database
*/
+ @Override
public void showDatabaseDescription(DataOutputStream out,
String database,
String comment,
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java Tue Jul 9 09:07:35 2013
@@ -28,6 +28,7 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Index;
@@ -325,4 +326,11 @@ public final class MetaDataFormatUtils {
formatOutput(ShowIndexesDesc.getSchema().split("#")[0].split(","), indexCols);
return indexCols.toString();
}
+ public static MetaDataFormatter getFormatter(HiveConf conf) {
+ if ("json".equals(conf.get(HiveConf.ConfVars.HIVE_DDL_OUTPUT_FORMAT.varname, "text"))) {
+ return new JsonMetaDataFormatter();
+ } else {
+ return new TextMetaDataFormatter(conf.getIntVar(HiveConf.ConfVars.CLIPRETTYOUTPUTNUMCOLS));
+ }
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java Tue Jul 9 09:07:35 2013
@@ -38,49 +38,18 @@ import org.apache.hadoop.hive.ql.session
*/
public interface MetaDataFormatter {
/**
- * Generic error code. This and the other error codes are
- * designed to match the HTTP status codes.
- */
- static final int ERROR = 500;
-
- /**
- * Missing error code.
- */
- static final int MISSING = 404;
-
- /**
- * Conflict error code.
- */
- static final int CONFLICT = 409;
-
- /**
* Write an error message.
+ * @param sqlState if {@code null}, will be ignored
*/
- public void error(OutputStream out, String msg, int errorCode)
- throws HiveException;
-
- /**
- * Write a log warn message.
- */
- public void logWarn(OutputStream out, String msg, int errorCode)
- throws HiveException;
-
- /**
- * Write a log info message.
- */
- public void logInfo(OutputStream out, String msg, int errorCode)
+ public void error(OutputStream out, String msg, int errorCode, String sqlState)
throws HiveException;
- /**
- * Write a console error message.
- */
- public void consoleError(LogHelper console, String msg, int errorCode);
-
- /**
- * Write a console error message.
- */
- public void consoleError(LogHelper console, String msg, String detail,
- int errorCode);
+ /**
+ * @param sqlState if {@code null}, will be skipped in output
+ * @param errorDetail usually string version of some Exception, if {@code null}, will be ignored
+ */
+ public void error(OutputStream out, String errorMessage, int errorCode, String sqlState, String errorDetail)
+ throws HiveException;
/**
* Show a list of tables.
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java Tue Jul 9 09:07:35 2013
@@ -49,8 +49,8 @@ import org.apache.hadoop.hive.shims.Shim
* Format table and index information for human readability using
* simple lines of text.
*/
-public class TextMetaDataFormatter implements MetaDataFormatter {
- private static final Log LOG = LogFactory.getLog("hive.ql.exec.DDLTask");
+class TextMetaDataFormatter implements MetaDataFormatter {
+ private static final Log LOG = LogFactory.getLog(TextMetaDataFormatter.class);
private static final int separator = Utilities.tabCode;
private static final int terminator = Utilities.newLineCode;
@@ -67,54 +67,35 @@ public class TextMetaDataFormatter imple
/**
* Write an error message.
*/
- public void error(OutputStream out, String msg, int errorCode)
+ @Override
+ public void error(OutputStream out, String msg, int errorCode, String sqlState)
throws HiveException
{
- try {
- out.write(msg.getBytes("UTF-8"));
- out.write(terminator);
- } catch (Exception e) {
- throw new HiveException(e);
- }
- }
-
- /**
- * Write a log warn message.
- */
- public void logWarn(OutputStream out, String msg, int errorCode)
- throws HiveException
- {
- LOG.warn(msg);
+ error(out, msg, errorCode, sqlState, null);
}
- /**
- * Write a log info message.
- */
- public void logInfo(OutputStream out, String msg, int errorCode)
- throws HiveException
+ @Override
+ public void error(OutputStream out, String errorMessage, int errorCode, String sqlState, String errorDetail)
+ throws HiveException
{
- LOG.info(msg);
- }
-
- /**
- * Write a console error message.
- */
- public void consoleError(LogHelper console, String msg, int errorCode) {
- console.printError(msg);
- }
-
- /**
- * Write a console error message.
- */
- public void consoleError(LogHelper console, String msg, String detail,
- int errorCode)
- {
- console.printError(msg, detail);
+ try {
+ out.write(errorMessage.getBytes("UTF-8"));
+ if(errorDetail != null) {
+ out.write(errorDetail.getBytes("UTF-8"));
+ }
+ out.write(errorCode);
+ if(sqlState != null) {
+ out.write(sqlState.getBytes("UTF-8"));//this breaks all the tests in .q files
+ }
+ out.write(terminator);
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
}
-
/**
* Show a list of tables.
*/
+ @Override
public void showTables(DataOutputStream out, Set<String> tables)
throws HiveException
{
@@ -131,6 +112,7 @@ public class TextMetaDataFormatter imple
}
}
+ @Override
public void describeTable(DataOutputStream outStream,
String colPath, String tableName,
Table tbl, Partition part, List<FieldSchema> cols,
@@ -187,6 +169,7 @@ public class TextMetaDataFormatter imple
}
}
+ @Override
public void showTableStatus(DataOutputStream outStream,
Hive db,
HiveConf conf,
@@ -406,6 +389,7 @@ public class TextMetaDataFormatter imple
/**
* Show the table partitions.
*/
+ @Override
public void showTablePartitons(DataOutputStream outStream, List<String> parts)
throws HiveException
{
@@ -430,6 +414,7 @@ public class TextMetaDataFormatter imple
/**
* Show the list of databases
*/
+ @Override
public void showDatabases(DataOutputStream outStream, List<String> databases)
throws HiveException
{
@@ -447,6 +432,7 @@ public class TextMetaDataFormatter imple
/**
* Describe a database
*/
+ @Override
public void showDatabaseDescription(DataOutputStream outStream,
String database,
String comment,